From 7cd91288b60107e09644e5671d8e748d42b97e6c Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Sun, 30 Mar 2025 21:49:55 +0100 Subject: [PATCH 01/12] define API for llm token explainability functions --- eli5/llm/__init__.py | 0 eli5/llm/explain_prediction.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 eli5/llm/__init__.py create mode 100644 eli5/llm/explain_prediction.py diff --git a/eli5/llm/__init__.py b/eli5/llm/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/eli5/llm/explain_prediction.py b/eli5/llm/explain_prediction.py new file mode 100644 index 00000000..1aee269a --- /dev/null +++ b/eli5/llm/explain_prediction.py @@ -0,0 +1,31 @@ +import openai +from openai.types.chat.chat_completion import ChoiceLogprobs + +from eli5.explain import explain_prediction + + +@explain_prediction.register(ChoiceLogprobs) +def explain_prediction_logprobs(logprobs: ChoiceLogprobs, doc=None): + """ Creates an explanation of the logprobs + (available as ``.choices[idx].logprobs`` on a ChatCompletion object), + highlighting them proportionally to the log probability. + More likely tokens are highligted in green, while unlikely tokens are highlighted in red. + ``doc`` argument is ignored. + """ + ... + + +@explain_prediction.register(ChoiceLogprobs) +def explain_prediction_client( + client: openai.Client, + doc: str | list[dict] = None, + *, + model: str, + **kwargs, + ): + """ + Calls OpenAI client, obtaining response for ``doc`` (a string, or a list of messages), + with logprobs enabled. Other keyword arguments are passed to OpenAI client, with + ``model`` keyword argument required. + """ + ... From b575dd13e074afc6fe49ad875c86e3d8a0be73a3 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Sun, 30 Mar 2025 23:05:22 +0100 Subject: [PATCH 02/12] preserve spaces and newlines --- eli5/formatters/html.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eli5/formatters/html.py b/eli5/formatters/html.py index 2f601ec5..0e96fa50 100644 --- a/eli5/formatters/html.py +++ b/eli5/formatters/html.py @@ -321,4 +321,7 @@ def _format_decision_tree(treedict) -> str: def html_escape(text) -> str: - return escape(text, quote=True) + text = escape(text, quote=True) + text = text.replace('\n', '
') + text = text.replace(' ', ' ') + return text From ede996a84cfb5dad655817d023589260b8ad4889 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Sun, 30 Mar 2025 23:33:00 +0100 Subject: [PATCH 03/12] basic support of LLM logprobs explanations --- eli5/__init__.py | 9 +++++ eli5/llm/explain_prediction.py | 59 ++++++++++++++++++++++++++---- eli5/templates/weighted_spans.html | 2 +- 3 files changed, 62 insertions(+), 8 deletions(-) diff --git a/eli5/__init__.py b/eli5/__init__.py index 1bc4586e..7a005c88 100644 --- a/eli5/__init__.py +++ b/eli5/__init__.py @@ -93,3 +93,12 @@ except ImportError: # keras is not available pass + +try: + from .llm.explain_prediction import ( + explain_prediction_openai_logprobs, + explain_prediction_openai_client + ) +except ImportError: + # openai not available + pass diff --git a/eli5/llm/explain_prediction.py b/eli5/llm/explain_prediction.py index 1aee269a..b4819494 100644 --- a/eli5/llm/explain_prediction.py +++ b/eli5/llm/explain_prediction.py @@ -1,24 +1,53 @@ +import math + import openai from openai.types.chat.chat_completion import ChoiceLogprobs +from eli5.base import Explanation, TargetExplanation, WeightedSpans, DocWeightedSpans from eli5.explain import explain_prediction +LOGPROBS_ESTIMATOR = 'llm_logprobs' + + @explain_prediction.register(ChoiceLogprobs) -def explain_prediction_logprobs(logprobs: ChoiceLogprobs, doc=None): +def explain_prediction_openai_logprobs(logprobs: ChoiceLogprobs, doc=None): """ Creates an explanation of the logprobs (available as ``.choices[idx].logprobs`` on a ChatCompletion object), highlighting them proportionally to the log probability. - More likely tokens are highligted in green, while unlikely tokens are highlighted in red. + More likely tokens are highligted in green, + while unlikely tokens are highlighted in red. ``doc`` argument is ignored. """ - ... + text = ''.join(x.token for x in logprobs.content) + spans = [] + idx = 0 + for lp in logprobs.content: + token_len = len(lp.token) + spans.append(( + f'{idx}-{lp.token}', # each token is a unique feature with it's own weight + [(idx, idx + token_len)], + # TODO add support for interpreting weights as probabilities + 2 * math.exp(lp.logprob) - 1)) + idx += token_len + weighted_spans = WeightedSpans([ + DocWeightedSpans( + document=text, + spans=spans, + preserve_density=False, + ) + ]) + target_explanation = TargetExplanation(target=text, weighted_spans=weighted_spans) + return Explanation( + estimator=LOGPROBS_ESTIMATOR, + targets=[target_explanation], + ) -@explain_prediction.register(ChoiceLogprobs) -def explain_prediction_client( +@explain_prediction.register(openai.Client) +def explain_prediction_openai_client( client: openai.Client, - doc: str | list[dict] = None, + doc: str | list[dict], *, model: str, **kwargs, @@ -28,4 +57,20 @@ def explain_prediction_client( with logprobs enabled. Other keyword arguments are passed to OpenAI client, with ``model`` keyword argument required. """ - ... + if isinstance(doc, str): + messages = [{"role": "user", "content": doc}] + else: + messages = doc + kwargs['logprobs'] = True + chat_completion = client.chat.completions.create( + messages=messages, model=model, **kwargs) + targets = [] + for choice in chat_completion.choices: + target, = explain_prediction_openai_logprobs(choice.logprobs).targets + target.target = choice + targets.append(target) + explanation = Explanation( + estimator=LOGPROBS_ESTIMATOR, + targets=targets, + ) + return explanation diff --git a/eli5/templates/weighted_spans.html b/eli5/templates/weighted_spans.html index 98d30c6b..3bad8ece 100644 --- a/eli5/templates/weighted_spans.html +++ b/eli5/templates/weighted_spans.html @@ -6,7 +6,7 @@ {% endif %} {% if rendered_ws %} -

+

{{ rendered_ws }}

{% endif %} From d3d4fa4644369494f70b6fc206af0701fd1011af Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Wed, 2 Apr 2025 11:50:15 +0100 Subject: [PATCH 04/12] add a custom color map for probabilitites --- eli5/base.py | 6 ++++- eli5/formatters/html.py | 40 ++++++++++++++++++++++++++++++++-- eli5/llm/explain_prediction.py | 6 +++-- 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/eli5/base.py b/eli5/base.py index 44834a6d..46f41183 100644 --- a/eli5/base.py +++ b/eli5/base.py @@ -135,7 +135,7 @@ def __init__(self, WeightedSpan = tuple[ Feature, list[tuple[int, int]], # list of spans (start, end) for this feature - float, # feature weight + float, # feature weight or probability ] @@ -147,16 +147,20 @@ class DocWeightedSpans: :document:). :preserve_density: determines how features are colored when doing formatting - it is better set to True for char features and to False for word features. + :with_probabilities: would interpret weights as probabilities from 0 to 1, + using a more suitable color scheme. """ def __init__(self, document: str, spans: list[WeightedSpan], preserve_density: Optional[bool] = None, + with_probabilities: Optional[bool] = None, vec_name: Optional[str] = None, ): self.document = document self.spans = spans self.preserve_density = preserve_density + self.with_probabilities = with_probabilities self.vec_name = vec_name diff --git a/eli5/formatters/html.py b/eli5/formatters/html.py index 0e96fa50..103e8a63 100644 --- a/eli5/formatters/html.py +++ b/eli5/formatters/html.py @@ -4,6 +4,7 @@ import numpy as np from jinja2 import Environment, PackageLoader +from scipy.special import betainc from eli5 import _graphviz from eli5.base import (Explanation, TargetExplanation, FeatureWeights, @@ -164,7 +165,8 @@ def render_weighted_spans(pws: PreparedWeightedSpans) -> str: return ''.join( _colorize(''.join(t for t, _ in tokens_weights), weight, - pws.weight_range) + pws.weight_range, + pws.doc_weighted_spans.with_probabilities) for weight, tokens_weights in groupby( zip(pws.doc_weighted_spans.document, pws.char_weights), key=lambda x: x[1])) @@ -173,12 +175,24 @@ def render_weighted_spans(pws: PreparedWeightedSpans) -> str: def _colorize(token: str, weight: float, weight_range: float, + with_probabilities: Optional[bool], ) -> str: """ Return token wrapped in a span with some styles (calculated from weight and weight_range) applied. """ token = html_escape(token) - if np.isclose(weight, 0.): + if with_probabilities: + return ( + '{token}'.format( + color=format_hsl( + probability_color_hsl(weight, weight_range)), + weight=weight, + token=token) + ) + elif np.isclose(weight, 0.): return ( ' _ return hue, saturation, lightness +def probability_color_hsl(probability: float, probability_range: float) -> _HSL_COLOR: + """ Return HSL color components for given probability, + where the max absolute probability is given by probability_range + (should always be 1 at the moment). + """ + hue = transformed_probability(probability / probability_range) * 120 + saturation = 1 + lightness = 0.5 + return hue, saturation, lightness + + +def transformed_probability(prob: float, alpha: float = 0.4) -> float: + """ + Transforms a probability in [0, 1] using the Beta(alpha, alpha) CDF. + This function is symmetric about (0.5, 0.5) and raises sharply near 0 and 1, + highlighting differences in low and high probabilities. + The parameter 'alpha' controls the steepness. + """ + prob = max(0.0, min(1.0, prob)) + return betainc(alpha, alpha, prob) + + def format_hsl(hsl_color: _HSL_COLOR) -> str: """ Format hsl color as css color string. """ diff --git a/eli5/llm/explain_prediction.py b/eli5/llm/explain_prediction.py index b4819494..049ed0ed 100644 --- a/eli5/llm/explain_prediction.py +++ b/eli5/llm/explain_prediction.py @@ -10,6 +10,8 @@ LOGPROBS_ESTIMATOR = 'llm_logprobs' +# TODO do one for chat_completion + @explain_prediction.register(ChoiceLogprobs) def explain_prediction_openai_logprobs(logprobs: ChoiceLogprobs, doc=None): """ Creates an explanation of the logprobs @@ -27,14 +29,14 @@ def explain_prediction_openai_logprobs(logprobs: ChoiceLogprobs, doc=None): spans.append(( f'{idx}-{lp.token}', # each token is a unique feature with it's own weight [(idx, idx + token_len)], - # TODO add support for interpreting weights as probabilities - 2 * math.exp(lp.logprob) - 1)) + math.exp(lp.logprob))) idx += token_len weighted_spans = WeightedSpans([ DocWeightedSpans( document=text, spans=spans, preserve_density=False, + with_probabilities=True, ) ]) target_explanation = TargetExplanation(target=text, weighted_spans=weighted_spans) From 892467689fe2af2b04e49fcd0c636c1421d1e879 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Sat, 5 Apr 2025 14:16:19 +0100 Subject: [PATCH 05/12] support explanation of the completion object, add tests --- eli5/formatters/__init__.py | 3 +- eli5/llm/explain_prediction.py | 42 +++++++---- tests/test_llm_explain_prediction.py | 105 +++++++++++++++++++++++++++ 3 files changed, 134 insertions(+), 16 deletions(-) create mode 100644 tests/test_llm_explain_prediction.py diff --git a/eli5/formatters/__init__.py b/eli5/formatters/__init__.py index 27a49c6d..a21200f8 100644 --- a/eli5/formatters/__init__.py +++ b/eli5/formatters/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Functions to convert explanations to human-digestible formats. @@ -24,4 +23,4 @@ ) except ImportError: # Pillow or matplotlib not available - pass \ No newline at end of file + pass diff --git a/eli5/llm/explain_prediction.py b/eli5/llm/explain_prediction.py index 049ed0ed..0af714b3 100644 --- a/eli5/llm/explain_prediction.py +++ b/eli5/llm/explain_prediction.py @@ -1,7 +1,7 @@ import math import openai -from openai.types.chat.chat_completion import ChoiceLogprobs +from openai.types.chat.chat_completion import ChoiceLogprobs, ChatCompletion from eli5.base import Explanation, TargetExplanation, WeightedSpans, DocWeightedSpans from eli5.explain import explain_prediction @@ -10,8 +10,6 @@ LOGPROBS_ESTIMATOR = 'llm_logprobs' -# TODO do one for chat_completion - @explain_prediction.register(ChoiceLogprobs) def explain_prediction_openai_logprobs(logprobs: ChoiceLogprobs, doc=None): """ Creates an explanation of the logprobs @@ -46,6 +44,27 @@ def explain_prediction_openai_logprobs(logprobs: ChoiceLogprobs, doc=None): ) +@explain_prediction.register(ChatCompletion) +def explain_prediction_openai_completion( + chat_completion: ChoiceLogprobs, doc=None): + """ Creates an explanation of the ChatCompletion's logprobs + highlighting them proportionally to the log probability. + More likely tokens are highligted in green, + while unlikely tokens are highlighted in red. + ``doc`` argument is ignored. + """ + targets = [] + for choice in chat_completion.choices: + target, = explain_prediction_openai_logprobs(choice.logprobs).targets + target.target = choice + targets.append(target) + explanation = Explanation( + estimator=LOGPROBS_ESTIMATOR, + targets=targets, + ) + return explanation + + @explain_prediction.register(openai.Client) def explain_prediction_openai_client( client: openai.Client, @@ -56,7 +75,11 @@ def explain_prediction_openai_client( ): """ Calls OpenAI client, obtaining response for ``doc`` (a string, or a list of messages), - with logprobs enabled. Other keyword arguments are passed to OpenAI client, with + with logprobs enabled, and explains the prediction, + highlighting tokens proportionally to the log probability. + More likely tokens are highligted in green, + while unlikely tokens are highlighted in red. + . Other keyword arguments are passed to OpenAI client, with ``model`` keyword argument required. """ if isinstance(doc, str): @@ -66,13 +89,4 @@ def explain_prediction_openai_client( kwargs['logprobs'] = True chat_completion = client.chat.completions.create( messages=messages, model=model, **kwargs) - targets = [] - for choice in chat_completion.choices: - target, = explain_prediction_openai_logprobs(choice.logprobs).targets - target.target = choice - targets.append(target) - explanation = Explanation( - estimator=LOGPROBS_ESTIMATOR, - targets=targets, - ) - return explanation + return explain_prediction_openai_completion(chat_completion) diff --git a/tests/test_llm_explain_prediction.py b/tests/test_llm_explain_prediction.py new file mode 100644 index 00000000..df241ef9 --- /dev/null +++ b/tests/test_llm_explain_prediction.py @@ -0,0 +1,105 @@ +import math +import pytest +from unittest.mock import Mock + +import eli5 +from eli5.base import Explanation +from eli5.formatters.html import format_as_html + +from openai.types.chat.chat_completion import ( + ChoiceLogprobs, + ChatCompletion, + ChatCompletionMessage, + ChatCompletionTokenLogprob, + Choice, +) +from openai import Client + + +@pytest.fixture +def example_logprobs(): + return ChoiceLogprobs(content=[ + ChatCompletionTokenLogprob( + token="Hello", + logprob=math.log(0.9), + top_logprobs=[], + ), + ChatCompletionTokenLogprob( + token=" world", + logprob=math.log(0.2), + top_logprobs=[], + ), + ChatCompletionTokenLogprob( + token=" world", + logprob=math.log(0.4), + top_logprobs=[], + ), + ]) + + +@pytest.fixture +def example_completion(example_logprobs): + return ChatCompletion( + id='chatcmpl-x', + created=1743590849, + model='gpt-4o-2024-08-06', + object='chat.completion', + choices=[ + Choice( + logprobs=example_logprobs, + finish_reason='stop', + index=0, + message=ChatCompletionMessage( + content=''.join(x.token for x in example_logprobs.content), + role='assistant', + ), + ) + ], + ) + + +def _assert_explanation_structure_and_html(explanation: Explanation): + assert isinstance(explanation, Explanation) + assert explanation.targets + target = explanation.targets[0] + html = format_as_html(explanation) + + spans = target.weighted_spans.docs_weighted_spans[0].spans + assert len(spans) == 3 + assert spans[0][1:] == ([(0, 5)], 0.9) + assert spans[1][1:] == ([(5, 11)], 0.2) + assert spans[2][1:] == ([(11, 17)], 0.4) + + assert isinstance(target.target, (str, Choice)) + + assert "Hello" in html + assert "world" in html + assert "0.900" in html + assert "0.200" in html + assert "0.400" in html + + +def test_explain_prediction_choice_logprobs(example_logprobs): + explanation = eli5.explain_prediction(example_logprobs) + _assert_explanation_structure_and_html(explanation) + + +def test_explain_prediction_chat_completion(example_completion): + explanation = eli5.explain_prediction(example_completion) + _assert_explanation_structure_and_html(explanation) + + +class MockClient(Client): + def __init__(self, chat_return_value): + self.chat = Mock() + self.chat.completions = Mock() + self.chat.completions.create = Mock(return_value=chat_return_value) + + +def test_explain_prediction_openai_client(monkeypatch, example_completion): + client = MockClient(example_completion) + + explanation = eli5.explain_prediction(client, doc="Hello world", model="gpt-4o") + _assert_explanation_structure_and_html(explanation) + + client.chat.completions.create.assert_called_once() From 8e57b4576599e4daedb2750ad686093d156f2293 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Sat, 5 Apr 2025 14:42:17 +0100 Subject: [PATCH 06/12] add docs for LLM explanation for OpenAI library --- README.rst | 2 + docs/source/libraries/index.rst | 2 +- docs/source/libraries/openai.rst | 63 ++++++++++++++++++++++++++++++++ docs/source/overview.rst | 2 + 4 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 docs/source/libraries/openai.rst diff --git a/README.rst b/README.rst index ecd53a6c..66670ab9 100644 --- a/README.rst +++ b/README.rst @@ -54,6 +54,7 @@ It provides support for the following machine learning frameworks and packages: * sklearn-crfsuite_. ELI5 allows to check weights of sklearn_crfsuite.CRF models. +* OpenAI_ python client. ELI5 allows to explain LLM predictions with token probabilities. ELI5 also implements several algorithms for inspecting black-box models (see `Inspecting Black-Box Estimators`_): @@ -81,6 +82,7 @@ and formatting on a client. .. _Catboost: https://github.com/catboost/catboost .. _Permutation importance: https://eli5.readthedocs.io/en/latest/blackbox/permutation_importance.html .. _Inspecting Black-Box Estimators: https://eli5.readthedocs.io/en/latest/blackbox/index.html +.. _OpenAI: https://github.com/openai/openai-python License is MIT. diff --git a/docs/source/libraries/index.rst b/docs/source/libraries/index.rst index f7bd9943..84f5cdd2 100644 --- a/docs/source/libraries/index.rst +++ b/docs/source/libraries/index.rst @@ -12,5 +12,5 @@ Supported Libraries catboost lightning sklearn_crfsuite + openai keras - diff --git a/docs/source/libraries/openai.rst b/docs/source/libraries/openai.rst new file mode 100644 index 00000000..9b016ce0 --- /dev/null +++ b/docs/source/libraries/openai.rst @@ -0,0 +1,63 @@ +.. _library-openai: + +OpenAI +====== + +OpenAI_ provides a client library for calling Large Language Models (LLMs). + +.. _OpenAI: https://github.com/openai/openai-python + +eli5 supports :func:`eli5.explain_prediction` for +``ChatCompletion``, ``ChoiceLogprobs`` and ``openai.Client`` objects, +highlighting tokens proportionally to the log probability. +More likely tokens are highligted in green, +while unlikely tokens are highlighted in red. + +Explaining with a client, invoking the model with ``logprobs`` enabled: +:: + + import eli5 + import opeanai + client = openai.Client() + prompt = 'some string' # or [{"role": "user", "content": "some string"}] + explanation = eli5.explain_prediction(client, prompt, model='gpt-4o') + explanation + +You may pass any extra keyword arguments to :func:`eli5.explain_prediction`, +they would be passed to the ``client.chat.completions.create``, +e.g. you may pass ``n=2`` to get multiple responses +and see explanations for each of them. + +You'd normally want to run it in a Jupyter notebook to see the explanation +formatted as HTML. + +You can access the ``Choice`` object on the ``explanation.targets[0].target``: +:: + + explanation.targets[0].target.message.content + +If you have already obtained a chat completion with ``logprobs`` from OpenAI client, +you may call :func:`eli5.explain_prediction` with +``ChatCompletion`` or ``ChoiceLogprobs`` like this: +:: + + chat_completion = client.chat.completions.create( + messages=[{"role": "user", "content": prompt}], + model="gpt-4o", + logprobs=True, + ) + eli5.explain_prediction(chat_completion) # or + eli5.explain_prediction(chat_completion.choices[0].logprobs) + +.. note:: + While token probabilities reflect model uncertainty in many cases, + they are not always indicative, + e.g. in case of `Chain of Thought `_ + preceding the final response. + +.. note:: + Top-level :func:`eli5.explain_prediction` calls are dispatched + to :func:`eli5.llm.explain_prediction.explain_prediction_openai_client` + or :func:`eli5.llm.explain_prediction.explain_prediction_openai_completion` + or :func:`eli5.llm.explain_prediction.explain_prediction_openai_logprobs` + . diff --git a/docs/source/overview.rst b/docs/source/overview.rst index b8fa5db5..6d612244 100644 --- a/docs/source/overview.rst +++ b/docs/source/overview.rst @@ -50,6 +50,8 @@ following machine learning frameworks and packages: * :ref:`library-sklearn-crfsuite`. ELI5 allows to check weights of sklearn_crfsuite.CRF models. +* :ref:`library-openai`. ELI5 allows to explain LLM predictions with token probabilities. + ELI5 also implements several algorithms for inspecting black-box models (see :ref:`eli5-black-box`): From fb631eb1dae1f7c674add1384b148e60e11f43eb Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Sat, 5 Apr 2025 17:54:13 +0100 Subject: [PATCH 07/12] preserve newlines and spaces, but still allow wrapping --- eli5/formatters/html.py | 5 +---- eli5/templates/weighted_spans.html | 4 +--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/eli5/formatters/html.py b/eli5/formatters/html.py index 103e8a63..9057e3bd 100644 --- a/eli5/formatters/html.py +++ b/eli5/formatters/html.py @@ -357,7 +357,4 @@ def _format_decision_tree(treedict) -> str: def html_escape(text) -> str: - text = escape(text, quote=True) - text = text.replace('\n', '
') - text = text.replace(' ', ' ') - return text + return escape(text, quote=True) diff --git a/eli5/templates/weighted_spans.html b/eli5/templates/weighted_spans.html index 3bad8ece..2dc4dfa7 100644 --- a/eli5/templates/weighted_spans.html +++ b/eli5/templates/weighted_spans.html @@ -6,7 +6,5 @@ {% endif %} {% if rendered_ws %} -

- {{ rendered_ws }} -

+

{{ rendered_ws }}

{% endif %} From a66b5ebed97434a7b068e76daba6cb66b9aa7282 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Sat, 5 Apr 2025 19:48:47 +0100 Subject: [PATCH 08/12] add LLM logprobs tutorial --- .../_notebooks/explain_llm_logprobs.rst | 553 +++++++++++++++ docs/source/libraries/openai.rst | 12 +- docs/source/static/llm-explain-logprobs.png | Bin 0 -> 56687 bytes .../source/tutorials/explain_llm_logprobs.rst | 9 + docs/update-notebooks.sh | 15 +- notebooks/explain_llm_logprobs.ipynb | 649 ++++++++++++++++++ 6 files changed, 1235 insertions(+), 3 deletions(-) create mode 100644 docs/source/_notebooks/explain_llm_logprobs.rst create mode 100644 docs/source/static/llm-explain-logprobs.png create mode 100644 docs/source/tutorials/explain_llm_logprobs.rst create mode 100644 notebooks/explain_llm_logprobs.ipynb diff --git a/docs/source/_notebooks/explain_llm_logprobs.rst b/docs/source/_notebooks/explain_llm_logprobs.rst new file mode 100644 index 00000000..6efa805e --- /dev/null +++ b/docs/source/_notebooks/explain_llm_logprobs.rst @@ -0,0 +1,553 @@ +Visualize Token Probabilities and Model Confidence in LLM Predictions +===================================================================== + +In this tutorial we show how :func:`eli5.explain_prediction` can be used to +visualize LLM predictions, highlighting tokens proportionally to the log +probability. In many cases, this can help to see where LLM is uncertain +about its predictions: + +.. figure:: ../static/llm-explain-logprobs.png + :alt: LLM token probabilities visualized with eli5.explain_prediction + + LLM token probabilities visualized with eli5.explain_prediction + +To follow this tutorial you need the ``openai`` library installed and +working. + +.. code:: ipython3 + + import eli5 + import openai + + client = openai.OpenAI() + +First let’s define our task: we’ll be extracting specific product +properties from a free-form product description: + +.. code:: ipython3 + + product_description = """\ + Stay is designed by Danish Maria Berntsen and is functional and beautiful lighting + in one and the same design. Stay has several nice features, where the lamp's flexible + swivel mechanism makes it possible to direct the light and create optimal lighting + conditions. Furthermore, the switch is discreetly placed behind the lamp head, + making it easy and convenient to turn the light on and off. Thus, Stay combines great + flexibility and elegant appearance in a highly convincing way. + + Stay table lamp is highly functional, as the arm and head of the lamp can be adjusted + according to wishes and needs, which make it ideal as task lighting in the office. + Furthermore, the silky matte grey metal support the modern and minimalistic expression.\ + """ + prompt = f"""\ + Product description: + {product_description.strip()} + + Extract the following properties from product description: + 'materials' (list of strings), + 'type' (string, e.g. 'LED'), + 'color' (string), + 'price' (non-zero float with 2 decimal places), + 'summary' (string, a very short summary). + Respond with JSON dict with above keys. Always make sure to return the price or it's estimate. + """ + completion = client.chat.completions.create( + messages=[{"role": "user", "content": prompt}], + model="gpt-4o", + logprobs=True, + temperature=0, + ) + print(completion.choices[0].message.content.strip('`')) + + +.. parsed-literal:: + + json + { + "materials": ["metal"], + "type": "table lamp", + "color": "silky matte grey", + "price": 150.00, + "summary": "Stay is a flexible and elegant table lamp designed by Maria Berntsen." + } + + + +If you examine the prompt and description above, you can see that not +all attributes are equally clear: - “material” is quite clear and +explicitly mentioned - “color” is also clear, but its mention is more +ambiguous - “type” is underspecified, we don’t extractly specify what we +want, - “summary” is quite clear, but there is no single correct +summary, - “price” is not present at all, but we ask a model to make a +guess. + +You may have noticed that above we included ``logprobs=True,`` in a call +to the model, this allows us to get a log-probability for each token, +and then we can visualize them with :func:`eli5.explain_prediction`: + +.. code:: ipython3 + + eli5.explain_prediction(completion) + + + + +.. raw:: html + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

```json + { + "materials": ["metal"], + "type": "table lamp", + "color": "silky matte grey", + "price": 150.00, + "summary": "Stay is a flexible and elegant table lamp designed by Maria Berntsen." + } + ```

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +We can clearly see that the model is very confident in the material – if +you hover over the prediction, you can see a probability for each token +– and less confident about type and color. The confidence in price is a +lot lower, while summary, being a longer piece of text, is harder to +interpret – we can see that some words follow more obviously. + +We can also obtain the same result by passing ``client`` and ``prompt`` +to :func:`eli5.explain_prediction`, in this case it would call the client, +and we can pass extra keyword arguments – here we’ll pass ``n=2`` to +obtain two different predictions, and would leave temperature at +default. + +.. code:: ipython3 + + explanation = eli5.explain_prediction(client, prompt, model='gpt-4o', n=2) + explanation + + + + +.. raw:: html + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

```json + { + "materials": ["metal"], + "type": "", + "color": "silky matte grey", + "price": 199.99, + "summary": "Stay table lamp with flexible swivel mechanism for optimal task lighting." + } + ```

+ + + + + +

```json + { + "materials": ["metal"], + "type": "table lamp", + "color": "silky matte grey", + "price": 165.00, + "summary": "A functional and elegant adjustable table lamp designed by Maria Berntsen." + } + ```

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +We can obtain the original prediction from the explanation object via +``explanation.targets[0].target``, e.g. use +``explanation.targets[0].target.message.content`` to get the prediction +text. + +Limitations +----------- + +Even though above the model confidence matched our expectations, it’s +not always the case. For example, if we use “Chain of Thought” +(https://arxiv.org/abs/2201.11903) reasoning, asking the model first to +think about the price estimate, it would be much more confident in the +price in its final output, but that does not reflect the real confidence +of the model, as it’s smeared over CoT: + +.. code:: ipython3 + + prompt_cot = prompt + """ + Before outputting the JSON with extracted results, provide analysis of the most likely price. + """ + eli5.explain_prediction(client, prompt_cot, model='gpt-4o', temperature=0) + + + + +.. raw:: html + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

To determine the most likely price for the Stay table lamp, we need to consider several factors: + + 1. **Design and Brand**: The lamp is designed by Danish designer Maria Berntsen, which suggests a focus on high-quality design and potentially a higher price point due to the designer's reputation. + + 2. **Functionality and Features**: The lamp has several advanced features, such as a flexible swivel mechanism and adjustable arm and head, which add to its functionality and likely increase its cost. + + 3. **Materials and Finish**: The description mentions a "silky matte grey metal," indicating the use of quality materials that contribute to a modern and minimalistic aesthetic. This choice of materials can also influence the price. + + 4. **Market Positioning**: Given the emphasis on both functionality and elegant appearance, the lamp is likely positioned in the mid to high-end market segment. + + Based on these factors, a reasonable estimate for the price of the Stay table lamp would be in the range of $150.00 to $300.00. For the purpose of this exercise, I will choose a mid-point estimate of $225.00. + + Now, I will provide the JSON with the extracted properties: + + ```json + { + "materials": ["metal"], + "type": "table lamp", + "color": "silky matte grey", + "price": 225.00, + "summary": "A functional and elegant table lamp with adjustable features." + } + ```

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +We can see that the model has already committed to a specific price +point as part of it’s analysis, and it’s condfidence is very high in a +particular prediction, but this is not indicative of the true +confidence. + +Finally, an interesting point is that if we leave the temperature at its +default value of 1, the analysis would show up a lot of less condfient +predictions, which is expected given the sampling performed at non-zero +temperatures: + +.. code:: ipython3 + + eli5.explain_prediction(client, prompt_cot, model='gpt-4o') + + + + +.. raw:: html + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Analysis of Price: + + The product description does not provide an explicit price for the Stay table lamp. To estimate the price, it's important to consider factors such as the design origin (Danish design by Maria Berntsen), functionality (flexible swivel mechanism and adjustable arm and head), material quality (silky matte grey metal), and intended use (task lighting in modern, minimalistic settings). + + Based on these characteristics, the Stay table lamp is likely positioned as a mid-to-high-end product in the market. Danish-designed lighting products known for combining aesthetics with functionality often range from approximately $150 to $400. Given these aspects, a reasonable estimate for the price could be around $250.00. + + Now, I will provide the JSON dict with all extracted information: + + ```json + { + "materials": ["metal"], + "type": "table lamp", + "color": "silky matte grey", + "price": 250.00, + "summary": "A flexible and elegant table lamp designed by Maria Berntsen." + } + ```

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/source/libraries/openai.rst b/docs/source/libraries/openai.rst index 9b016ce0..63af0f99 100644 --- a/docs/source/libraries/openai.rst +++ b/docs/source/libraries/openai.rst @@ -9,9 +9,13 @@ OpenAI_ provides a client library for calling Large Language Models (LLMs). eli5 supports :func:`eli5.explain_prediction` for ``ChatCompletion``, ``ChoiceLogprobs`` and ``openai.Client`` objects, -highlighting tokens proportionally to the log probability. +highlighting tokens proportionally to the log probability, +which can help to see where model is less confident in it's predictions. More likely tokens are highligted in green, -while unlikely tokens are highlighted in red. +while unlikely tokens are highlighted in red: + +.. image:: ../static/llm-explain-logprobs.png + :alt: LLM token probabilities visualized Explaining with a client, invoking the model with ``logprobs`` enabled: :: @@ -49,6 +53,10 @@ you may call :func:`eli5.explain_prediction` with eli5.explain_prediction(chat_completion) # or eli5.explain_prediction(chat_completion.choices[0].logprobs) + +See the :ref:`tutorial ` for a more detailed usage +example. + .. note:: While token probabilities reflect model uncertainty in many cases, they are not always indicative, diff --git a/docs/source/static/llm-explain-logprobs.png b/docs/source/static/llm-explain-logprobs.png new file mode 100644 index 0000000000000000000000000000000000000000..452d9219a5c7c26d5d9831822c0b22c5dfdf2f43 GIT binary patch literal 56687 zcmcG#byyc~^fowjgS3Ejw}f=3q;!jPNOyM%h;+w??rxL@DJenel9cX_9e>LA-M#j& zeRp>*28OBUJaNvs&wb8(P*RjeLncH9002$qwS)=)z!m}kR2m{Yc;;1>TrvP4*I9~- zE6Iq9lPNjcnORz!0)Tr)Vlsl7>MUO9SQ~dHEOY?kYou2=q7fnhJ$XGKTa}9y2juiV zLuBAmgO|2vDh`LRLny)njiF4+!;OewarYFTDJ%7N1)@#41xTi(HZ6JW?w4ntZgkup z)QxFxl{dfuAzksWzZ)_D1i!2;Q?l_XX}F>TNMR8n(0t=iS|q}9-RMY2(8UVTeoYvH z03nf`_Q#Ez%dN*BYH;05LVyzD(1$=KU-+SZWzr3~{8@lZT4~Y3q0uDDF0Y(L5=Fwp zAS#~L-8l-opFo`W(+9*gh&3t@@NG>d9*S%p!bcgF9kogRX~dKgFT9s-D`zh&w&u0} z32(y4_;aKo<7@2?euMFMO&K2|G6eK%18IM<+rru@CsAcKojoo@KiUxdt{vn}DUMgo ze=yuG{%|lmSuYuYj7K|)mSqxaAmE#!YVr^p|84A7uY#k7Cy!ekoWP-oWY$@WXnY3M ziF$8%Z3|{R)3+0(Y?I%)FZ;g8sGxhrd3-M+G_j0D7L*)NUmsKW5T3RELU~G0(#CQ% zK$Y!92uorJ{Z9&9fa^Ns^Tuq@P9Q@>~l>txT~K3^Ft~-U2#j?lXBv4iBQ2<-!$HzO#po z8^4W;uqvv*J_b04Ci0vuWsUKmMHUf7=2hF}(>QVrzqVKV!>al7jYE5OBBnqw*0W2m zAPY=Hz=*sI2WkwUR)TP1pz;maw9qwzpp{^`g7zV(YW{5y_!scfqS!oeqd{^jgxj!- zD^M{(zDHQ70T>3bfn)*_0TwZM`537pNioE|feYlMbco17Mmex_1X$wi7QFR%41p0j z?8R86A-(~MVp}2_0|u-4%2Cq0ISlc>g~F^Nx(j8Xns>L@3w^<4 z2`O6FFFk})bWWikc%p5Gk zQv4E9jYB2U?@E&?1(*dj5_x)S^`RNj;|jm^Jo$XE@9EmY9!%2?I4?L?>(3{5iWW02 zl$}}&iwc$Ml@B8HYJcH)xTrTs9F?(|-J7 zZR4xU*Gq+P8m4%hcw(A;8d8PS!imCN`E+@Nlr(u3j%4kAZwo25)0e}XW$d4AqB+b{ zi&77{4(t7yUofdK4Kdqlbtgq9Q4B98z2&Ip$S@B#R~ueXcWE5mY{{ z#jV|uWhua6yP1@_&(Sfj0WPxyyL%g7hOQG% z&GDs^t&{Uu{91Kd3R_5=iBm>0m^0e!T(i@2h{h_uD;$j4zp;OueS2WI(sf#V;{MBW zo&Ko&!h4jqV=AGB9yG)y+yU%*Fy@vg$2j!CQQpggq(0id+Ayc7GK4?DiKD*vw zcQ5Xet}{>jZxC+mjyrmOY@D<-EaU)1{P7yD#_i+kUtP=nRk{|sb^UE#s7iQ=yq&DwT*&Dm z?phPig7x(~d(WI9v88Tfq?zFD;2@;DF9!O!c094^&5RD=W2=WN9XRgDjY#kzw%tan zr&LY(zw!f2n${}@gXl@+d;g2m zVV1OubRW5Gn6T14aXCFbx%EA|&8ZY=5z(+s#Wu?q?8 zE|cVvq?XJhx+1E4Fu7Iz{bre)#%6D3z1eciY{Be!GmDu-_kDzVq*LT=C5LOzBM*H}p5qK0CWsET20x8C;GCW_j%} zIx|KvwBIq^T-z4BGc``MR3Fq|SjuP{yO7+sS{|Ewr(5y%t=j#$|CRq2n)1SmpCyeK zkkKv$7yHa|+asmBoY9`8u=L%E9Y&&UG)d`B%-LA{4ywEFPMk*;yKFyeZq{h}-7+DW zbk@Oy$z15K3OEPTjnh9X!e?09IN0zWmz`H3C=z5a2v<0GP7-`iz2jIO+sq;sqUF8Z z@jkh3{~1z&!P2YOyofjV(}C^i*M?ksGE05k{FuJ$XYZYe;ly}NZO!ybUVW=C2ln^9 zcIX|M+xB}FUsj#-S|(fS3anhGQ`1Sf$CgjpU4C<2UNjuEEWQ_?=pNO!bZ+g<%5nEr%s0172itBH#|+cX4! zD#{_RW604ApI$78`cUKc- z1Hi) zn76XLwiZ5nb?|O^!D=UoO8ncLi|N_lLQ(wN`1sgfN$%$_45De1I1rkdyuc0JQx_nDCe2VaoOJ@{A)dTu$ScjO)JJ(iFVBg@0vQV2LYhd zTM_D{iiy}ITO@#?9RnhONbh*|Lc4`&4Tfex1!Jj zf2O^S3uJeKCcLq;__kQCVLV}XWp2mcDG#mfO_Rb+4OLa}y}3_>Pm!(Yy{0SRI9K#E zI1*u}a(UqJjPu*6;=gw|^I)2ISPv#^444-zbWN`ZSYU`*DvRnC zyeShHr0mk&f(EuX33A2@@I?s6Y6|N{x^>7WUC91*Ax$1TE0Hy+(Q^-qB19jmC$5~H zH)$P@9KjUvJsb|$-zoz?U`db#Q3tBkLyE3$vV-YCgY|A-=rs%Gju9HCx+k5&(3Feg zhnT)AzQJ77F&0`0>1T7Po)$ySK#$UsjKS$sol~o3%UewpGizbK4tSe)PL|y_?dJ0n zw_&cZeBKVjuQvD+76h?3IZjoQZ2aml7CS5h+&UO-TOi7vj*3DNhbd4lIW&eHf@R#I zju9^_IrDBx2!h}AcL+H-9USn$#hPZf5HyNF6pv@+SB5dNyqNqf0UDTPHC%ZoXwhoMEm4#LB6Wt^ zH)4}$UMpJDr1%NY{vyd33eyLnJ~qqSTd)7M!l^(!Ay}wFaOVzOaEptQx_rPh6;6Ze zahOGPzFt`{ke<#{i1ESl%(jtU;+D;!pOm_Av1!cP@VXW6VhI6;J>UlT=n>M`nKRP& zt2-u84xzXOyfQ4>9WcxUJ|hP5Ip}5G)`6F5CzGqKX=zFYS0&9rDC*c$AaheOMA#gcI zx|V;d^?CVbM z_>r((Cdw}ZquHSsuQ(H1^KY$6bBBX`9`fg(q*QQ|HZ@?lWRx6N3l0?S5)z&}KGIE3!aHfJ$`gw4}^>CcHGL5i{^U?ullX{R<+33-pWsl ztYTfkn@?)u%+CO$DniGje!ML(Wvvr3^ln6Y$&`9rL)we&no!W9xfQR{W@1TIR<52l zzQbjG;mt$iwx{W+a;kCGKQSyMq-_k|hj$kTZ+wAHxHsY2Z1WQ^d-(nQc{JJvB(4n$ z{$YuDRzn@~!TaK8i!{^pXW>?T;yThNHxU93ZV9OKT@L`&r-bU!`nkXe=}jr$i#kWKE}>PahlnW5r8w$AvHtV61M>irAqbN1iz5LUnpu#la^?*K90}cE~n=tuUlPQwV zP;sikzOhJ7`hr{=g}H)=er5Q`L0EI|7gIcWKU|RA5ep6YB9*>)HB~T2k3?gG_mf@h z2`oq_P`c7(dlg~Fslo!qC$A`N%Q(v*-04yaX6rtmW`_SWYNpZ6;Sq-rqI|V-U#?#{AG)DgquwW{{OBOPUJT)jy3t#y3I34Bl zl>dBilz~CS(*^dx0Q#p3Lx}&6ANZl=rbV7jT~xPdW?1p^V&}O-^|Ho81q>pjm=pAF�>yBkY3m0(x&OwY8401V zmBcavwg5E4>w0R4J|8W+ai3OUF6BIkYDN{f0RL94$k{5U0Epon!_$}$E(Yhmu6-v4 z8O8tx_;|xa{~ImY`Ctl}xHr&L1OQ}6^NnoClQ`SkFD>6@z=pikmVB}i6UzR^hfmk( zC~5Fg;%G|no&wso7~lOw|3Uovzq@>}k>zKXw?BGwu?%O~n?9Z2-p(Q1MF(B% zwDHK%o@*CKI07P;%02FZ9^VdJ9Ii-xz>FJ%3aeO~xX&0@ForG^cTG8%wG-B7%y^*~CPYlFY zo1F;|b;y;Mddl%PF`PvmIN;g7DCnmYTo?b>U0gY$?n!m0l-Y~(#MqUnPmL1`(%_`1 z;9BGg+f)5#Nk4Y=eox}~&HHrIM{M!`CNTjs9vrlSL5BZyCK?_bAbMw65pn`f^~mgC2?fUl_nUKTW9YSa8@$s)h~#1SpcC`(AUL2~ML=&m2fonH|j#g*2pEJ^w3_ zG}OiaOY`w-(7$Rn{x=&Pzp+vnd+UCF%5#r3m!lIFD@+4 zlS~XRmXm>q!ImU6&7a+$`;kD9=%ml@sxtIVO;^jUTWUa**YMh*Ll<$lW;V7JuIF6I z0!Jx$H>o=6*Nf&_yexZu?U&`5148u9i*O=_N});D)B^$IZaFU*DCb@8O9^~fooN!X zeRNnDNYnU3%>UU<^y@8`6!liyHk827oXW}c9SzrZ!OCe1Z~zBso{9@ccbF2-cm`TM0czh|};0-z??2!tBf zv8!P8#4*vG$%0{(Y#nmzbz#&Lr44~Cibol9iif!5y7?*F#38e`V4)X(q9nm*;SSsw zljq4M#$VS?F@7JfRc60mw@ULvVq#s2SYQ}7Nr%7`Q*(vnN~#tRc_N} zlH+0YL#y6rj(^zFjvLEd!o>7vG-zZf8Qw-i8{t z;JYgf{2ehgrguyO&ab{Z@-&4slc`%Y?yShOll%%@Ut~J2@YfcnMCs|%n zQ+c$!!GnZTw@3`+qKa08pOwZeP%O;DgNp)^ZXQLYNNaQUf00gvW6q(klO^Cd7;wKVbj^U$Z64G=S(cO?JqEeg+z2dDuy#X)2 zEevVY$@bPIbB?lq!&D$c;8~IkbqIxjH==K0zLS$_8eSF;!NRJPVVa=t*4+x^q!Z&_ zMs$g#`54L3`hSffRkqEq;wL2wZskCtAoIi-fx$Z1*Ar>e@-6QT#;oBCqI?tD& zUK!2Bm<4lDMLMpNL@=QY#JVCE00nFwx&-fOe?buMVdrIPcc1?zFM6i3^e;Maz5*9f z$;>LBIBP+Tcw^jo+Dx4%HdPVgnk+}DuOKP&@*>8xhL?%iD^Y65(WWfUk{XmKIpnc` z#aVC;Cs{x{wL)xZgPQKa*X6q>+%3#@a}`PSWrafkjVQec|Fr*iFSG9YA*`mc02JY% zswLv4^6SF3*fGf_^_=a^q2}|8rrYuRGBbCK-oql6DacBzugHzk9EoKQvXD+wJ& zy^us)upo{(vVn0yQGszJ<#_kn;_kqgr)e%AhX7EAmbI;vaA?lFXqyU&rrPIxJi$sR zuqXlyygp+P)6XeQxNOk9_!(+#cbRj{fBP$`#SPu4zlKOmm=2bS|EQ(;9LDYiSC!T73MPpYs=U$y4TAKt^$fb9_y@9|kQ(k^pWxu^%71u4VrR$yvuj;su*=!=iTB zLx&6PTl{+X@kbSWP!rT18_jyN6dl4)1}v_Zgl3la$sBEr}g?=(&`U8#>?kIO7B+Xp&6OTst$dzwctJ$uOMX2ZsSSE7!r0WcLScn2x zNHd7`wms{SKs0z*w?A+zMbBwEH_#4F$9s(1rB{^jzTbn7A z%KhlqO0we&N_Q2G&yD9~wm<^rmDkFGwe)i2TRPdgOn^J~e`II=;Uvgs&#o$4VvpOE6S#l}E zyD)B{WJ1lGu)l7Y0V0qdgy|x5(DQ9?60xnNN;mMwK}^WZ0LXo zh=3Ls$Z_mBLy<=k|7M~r1@cy89%3H3LkF25Uxwv!&-r-~z9yP+zU@GWF{xkD58InO zrGTn#-Yrsx>`b4=`gr~tZ;n0d_>PT?h$wPBwh1o8`mJQ!c1*Cjn#XT%$Zx4Zzdnf$ zLivb`gi=p$DAfMY%8&mydI_If%|XDXt9}T2ix!MTd~+OR169Ob_?@Od#B5wGsb(+N zGZ<>Kd*I{eFcfB*fpcaj&!2gH1+8T?XX)CU$=AI<)i#f$+MGF*8+DVyrRH$RgwtD} zvls7VpkpcJ(~S0NQ9q1&l`H?7MprWvKTChY_i|Z2obQB_ilx3>2)u0j5~Q}YUfug9 z_VGUQ0VTQVjLPml9(rWuE2Zq~eic-0Uv&O?{+U$J^8+&Rj-L{W`F|d*8UO}DO=LJJ z&=An6p~!*(2ifh^!fRl#ZW%(}7L&YXgS2mCVtv52#Oh z;T%Lu2G|rwBp-08w{MLRK-cpw2%(iXG|$E2?B>r^YJ)wV$|?Qfop z4Zb8{9u0r*SXu~Wu6JAPpr8-re|a_@jBQZhrjx~0OT?@DF^1tnET^3W5@dE_t+m-r zW#%kLTi|j!am4ng;_@@o-i)BZ6+U}kBpyI-E>ASXTSUe}_o~z+NCXqMJz2rV)+{qOib~Ye(m_< z--ncGnLZJdm}#X6SmbBseDY3$p``kf@0ktm3Kj)m#rTAZ-~Lit}slZx>hhmDwwP? zL^YJ>JcQ_cyQ*a#ViiQ(E`QzKb!JSs&KimK?^3KNRP^GdsqBogL?HUn?@!@p43Sg5hDT=j-_ z+4CdGi}92HkKnnKlghX$X$;YB6&ClR#Jyc5zkhRlK~S!peGEl$zM*MCk0#UcQ$pzX z`YFS#`L?{T93MY#H#d9&+ zh`GNJT&mwV{?`^Zj4Jl}+>Qun2PlNoHMIZO*)&Jygjoi<5r!-D)(Ky``{qwM^%5P0 z-RTTc-;}AZsp#sFi{}QvNlnCt1(YB;mPcNo(l=MWpmw97slX$~-j%`S zhgE)CDtSR*{6=30GfpnpDxMgCj5D$wF0Mg_6ut%IbC%KI*eg9xtj^E`8TFjhg^XRB z?Hfssg?Zwb|6x5xI;VL0&@rYG-W#R{a#6KPpnk274^f>`{OgRj2OQL>g|m^x!TjPt zM=R5BuAoPP*Pp*Ym>t3sElr&f=UoA? zZC~_!UVgP%8*~o^dkd38?%k83p{sN}urM%Ig4H^?T?@il9}v!7Q(ftNIcjhxj~)*V zl);+%+{ou|&p(*D?ke^#Jf(pzqi-BOGE8u={aQHZ{Br40e_KSF&uEabBMpTR5 z1WE6bdV00_wbUNJsoZyc&YG4O5F{82Nr`#fLxHY6JZb)-r_IT}dqV-gFony+Z#tq9 z$ENri!DYs02X9aDdnjxOAWGzEa<#MQfb=8a%JtWq3B|Pj2}N|cSz2-nG%^dcC`%2Q zzZ#nM9DO4lQvQm!x@`#suxK~LaAaW+`?=*v#usRB+t_m>wpCnrVp(<;I|lQ(6R;O= z$TtDqV}xB_hYF@)N_};MLWf;NLB8fj5_^Fk!BEF0z%~i0T>}GzL*1Xbr?4f#kYr;a zP=;T6A61ZZgYRno%kw)6;G^b`?z<3gP(4;FU5r+;7h{b-g?uj;yU_Ni22r&^Ht9#r zL#ZXxzLzW*=LZh9wRyM66SHgIw$|TV>3Kd>;rfh$m5ku3N$EIn7a^B#Ug*h$;Jn;z zdR2_lloS1qv#Y`Bnf5Bzy4^YCGDOk6dpWOAJ~Cs~<~2~C$mfpH3a>zv(Ar#aAuR9h1 z*}Hhp88ZK2Ef0z(v;o_=Ig_V* z)v5F&*S#6(iz>1*cpV4m8LH&2%sGDZaUuZM*0IZ{61yR*XC=!@0mqTdTX+VWr~M-D zj3Ykl2g-^(Bo_Rk|LD#l61JYx+i9V`=I-}zdDIX6GAv6WK0@(%8}}vCh}iigitSj=j0X%lYGrO=0BGAK;6 zLx*zVReFsibe}z-vywrVilI$O8Vmn)mmxuFm*R$PfSzL@h8#86VHn zsjOrO7FqFgJ0)#3B?SoD$EnX4h?nl^xvuP~yV`lKqDQ*;%l6Xw$onb=*MQE(I#t$J=&iie$tmWbKhxkdSSqS{ z^Ao4PNTCFzr2nhxm@rwHd7}6!{x>ok|GTLWkWSq`V#n{!PrAI@g^AZ$T3z2cb!2}F zC!2#5f7*QWsoK@i4)L$Whwl78)r64$?^K?crudQh`wOQ0elH(|%9AdbS_$`$PT(k& z*EINMBH|bcmgnU8L-68Q`H?|)?5G3}`^1JqfvJm^47At&A}_rcaS7pYRu3r*XE~3L zvqZfQQ}KY#$Itbs@APa_ztmPOz}=G2o>A*$T+G!DoNQ{g*4!V!Jj3Zc7S0ZoSxy#HdZEAk8OOB;pn>uR6=fH5`?4;0@yvPEY~Er(3A=J>sw zavI!^r)jYE^8?x+0GypG#fXo7zk_iQiOYU(RLWq;7w{dJ>?|4t<&DAt+ska>0mMNA zf?Qrh(cC01n$z8w?cC8Q%R~rvvta>8Z8c|kxa0D_+WwUr{r(da73T`F{Uu|ynY0sX z9#3__kP}s|R{8wuuB6HwzV|JH`mYAjI4cg!2nTQcT&lG*uM5$*0BR1@tMha9c%7o( zg$509_u0H+*_ztyx~VwmLi5oW6oxh|(NBmUwc$>^RcT6jYsJ0o z{$fF8&q6ShoD+F*r?ic`YC4o+IvBPNIwrv1pvDf_Dp}iMTB_)^USalvK1YoyW6$(k zf+nWxW1ZVO!2j2a^ZcLYd^%Tc?{TsWpm4(M42JAbIu}XW{knTNjY~!tGFohT!GTt; zo{Ky28p}7i42_Y}{hKq%EqPS|qbe4M|83@6Iz0!-geU{6FN9S+Y}9I4FILBfQpj>n z=pIITCacfms&q6%bVkGVez^G_ZH0 zIW};KKq7hjw9!jYi>4x8k5OqSuvHI#@TUwfPxX90)V;-yJi*vCyAt|&62T@5e%1*s zcI|^cQrx>h$TJ9g-!!>4Ye~r$?!Q);P7tKSafxO=oCS{4!j}P^53Zy-?RBiSeq*De zdqS?_?#57&7~be$lI$X@9<&qb=DiEts$=>;K~}chZwCN#4B9XtlM49%S6^iO^$x0Ib&?NdSsZWWfSxL<0M=YFWoe;H&SkT|T{lsawk}c?PD+biQ8I~M;%aN0rLwY<% zMSPG#;SVzJdPMu4Csznky$i>Aj>8t$Y;9Teah>`JFy za&N^UDeA9&*2c9qVsv-osywmI3>D&o>-Y2;BXU_|#UnfY7}<6`MF^QkaJgMCGj#$c zlg-wahKCFZ*5s{WM#$HK0J6> zozuW(h^|+0t_2t28G(pX`VgKdFl)P7#HR+Ye@PgjuV|B|;xp6W%;hfLl<+c%rkg0a z!6pccS}KsUef@;_x(B~?IIrY4lP!J?;icHlvD&44L&(*DG38Q@z|;wSC6?-n}>ViWAPQo0o1{srssF>82Lyo zW%Y?Wf;a8->eDI(>N!nG}`ro`rAWzXw!?IUwzGX&?^4qQXkNKhH6RVR_IHPWr^%vLYl z<+S1bQWtm?aEHNKnlb8Wx!ji)*Sxv@YF$ZIc_k@MMR$I#dg?HyNefo7yxa+KX!1B7 zRBu2Mr3ao{+pn!Ly5;N!Jysj{O!a_@<5@mc99Ap-($g~~e`@rYM%t(q4xxD5~2Mvz>RsR_>gtov`Q+{BUd zDB-t+iphV33BqM?N;Jj|rLG`NjFJ|AsiR9u;i zk+s@D3EV4Uklac26>nKOeYpC{N>p7n0s}N(e1l%N-uPZ$HR%oqAEf1{!0i1~DV<}C zBlj*6Zs46^Xx(h&0mkjbg&C{sW3J^?t$?6!{aqPRQV5NjU%~Shxw_R|jFpu-o9zb+ zpq9$`j|L4|dc+>u^i|=5{hK)2La%S1g8_IpD^myF;vxSc9Ck70%g&iaK+)7yau6l| z_KL=x&wJ+59R>b90u;dSH|PIr^5j=2B^Q|SX)V%)3N{NYZ>~m^v~q-usOGt~p;c6! zcecJ_v&XmE7E&to_~(Wva@g*nRb`%r`#Ghx5N!Y(?46}PZFPpqo*+ZLWE_wsYicHbN%VqtVRQ=P+d@e2Zv z*yJ@Zg<#+%IAtu!ulh;b?=5Np@7ekDZ-7HKb0TSeN&6NTN&63=j zxkooA&4z*3nu+9xR@5@5m)rQybG~UvI@%IwrmAMCvInYkwtQBE1tnEBx6w(gW}1ie z)@pxkexYh+Y%yz3eD7;r<|bODZs40r($QfGf5yh^ADwLzOa;UGFpZq7*?oYAfu~6X z1cx5&zek6gSJX)fA$>>ECS}g_%J}?_$@_dY$l?I}m5Cl6m+=fI($p=o`6T&#W)cfo zcG24w)3y!bECfZQCG@{JMJIg@V=FFQsR{>9)t>6VoZQvB|wVPQ_p!(i_&O z^}^bz6eVXzmZj@Eyyp2f`z8-I_f9w`1v(LiYie)T!>)k`IbTj94i?4_cY4)Bfdt8k zD?O|>VYDwe6$|$^WfSz?k>E$Mw3ed6f#LMafDGfm%o8WuWNn#0nERCz$a+7kMOUW| zrXCZPzkXhZdly9O3nLSHzprEWu%nhUE?)LR_UOP(@+Vrb zS3V$e9{(y9GW84$i+4W2#X{n}`VGXc1NjoZ2{s~xLy%cHs5Vtqo+qT-R~67b=)W?1 zIDT%D9VgH+_F4^|TCJzjtJc(5IjyW~wEt6ACAcpiWDw&g5T0C9Ppk|yrsO31u!*-# z$wb>t(I0(%Opko~i8Sb=@H-wOrJ+?khaaFC^1n#9c}SK;*9Pq%yixm_`Wxmb6go8R zqeTWBUlv+J5BA7|l-T+xZA}R;zgbu6#U0W}21>%ql zY5OXQ>M6YsZ`X>%?I_QAFsd58a{Cd@FZs?@YJ*LC|NSKj=UZ;V5a-zxF^ zLN(f~!@}0W7>t&&-JiuC`b`BQ1KnPbDW@27wBOj;tkSc$pw^E5k*%{rP zE1Z0GUH&V^xmA%QD0O*}!L`UuNNLHGwAr3+(`0~dw(Zvocy5&feOc3S%3r1ZgcNO6 z2M@CJwmkeUtSu!kt2uC}z9(D6X67o001=Z&bjTv3zsmE=q7o;_Jt`Z|PHjIXa!C`r zl{fRLNn+pJ-sB2qhgL#8+gVJMT>^`K>WbbXDE$pX^GKH0js7G8L3YO=TiB2%`+g+F z&>O!1U8xxP73HXHQ99AS%*HJ74jb+s3b0CU3s!1gl+UC%KIZgpfwTfhKgbTp-r~B> zLu-Muf4t4G*uLmm#bet>x=&^p&sRpe8ujSAx)uiWZFbf*s~?%Uw4!8R2=@oSLaDPq za76i*cNXVyWJl~E?6+2bC&OFh+`{jDbRO1jB{wqQ(ZLicHHK2iIG&HX(929}1#5>Y zU#9{)?cbcGSEK%X1{bX}O}Z6bY@6@gwq7AW;$PM$oVZ2#qXWn^GHv05S@nwnd!c=K zK{87AsIh7$qPdhM;vUN(DMiEzmPsk0TU~MSBnjO+Ca)5+MIx5*4SpjS)k9yi{(V1- zzLo0P`R@)rvmpb+8mldro|JXlvw9Rux8I^Kh3Y^PH|H?1qqlI|z+U|Qh3(a;pX+LL zBz9P=0xo$WYaY|b7fcZ&IZ3r&ddOnqUPeEp8n>dWK3Dr2>*tm@m`J^%A=}EOI1`*l zC*#bh+TN4}6X;~mNWOBb*wF(=P3}_Kq$AlJoJ@A^7-c&}DqF*`-dH%Rvqzltp^^%2 zx)`nYYuF00*_tC9tIe}g2^*F%HcH*rO1bTmkzS8;S+!Aeir@OEX*Z4BA`wXU33`+G zXt@sDy76OHF^1faJ2cLC1KoEDcfi7MUs)mjn}G<>qyBCX)Y2;H$4}O!Ih2rwu>KTF zt_o-x!?-<&4YQe}vDDgHrQ1qj;%r-h1F8>Q$+uG0doanveq#S6`OdmKr(WK`NrCCi z)y1s{)UCR(Qw1bKIHoPUsAj)$LXmMoe+JfTdm+2GXB*lbWSpEbiACttuaZrQkvRv1 zC|#mU^SW9yqX)KjM;YYjR0mfTt7w08%L?LY>^St`BeSib{_15lt}pa+ylJ z*a$?#p|%NZq*yPwX%lQF{XWXWp9285hZ@{E*?KLdtk(VoliC-?V$%rsM`u@t^ZUW3 zhQE}!v=b(-+s|XBXtA04CB|BuGI4(-)4UN{uJZ+;%wPWI`v;u7iCqTAf*L2@>4(KZ zye>bl@Na{*ay-)SV#_H5MDQB(^-R6+9L#p<812~p+4AC``KQw>sxu6DLnk=w@6;P9YR$(PV%~<@ z&Qg7E6>O=THfwThScG{o-U+b@=;poa{`iKoYm<)?DrTWzA{BvqshX&rN?gEAXxzVj zpZZ&+wG2coc*|FH~&%k@kgq2I+SK{ zQz99F!l+8SPI9s~JY3^0W%0RDNVxs~lYP!3>AmVk4}+%gQE&>tiGODd;3Y9`2LV1L^eY)z-!0M`Hw?hl!8- zZls-l?X@tzx}F}hQk2(Ju;ozTPq^j9&_8`W=I0ANLGo`@K{yOhI!yUm_Wn;P1}83G z%9!F_Y#zfnnUw-Myzt&9*f=$h_uw}ao~<`pV6nm4tA>}^i5Os&l1UY8Kf=9z`R=Ew zq7(d6GK*Y?E|7d)nKKF7Uv5KmzOp}x8#54MBrQxd0||v`GXyXnB7z;liS*XV49i?k(iQe>fp!%*P4H`C}sL4nsrrh0ZlF7KD(p%Lco_ z=v2qk=c>ocN2`Myhf|NQCp>Opih~Y)q8L`dLHrd1x~VO{|jizkgq=BIx`{ ztl;Qt1ny*i%Tn_A2W@KeC||;nU4A!mzRQOau?W}S694EAFbTPAau$Zo-Oq4ii(C7KG)Y<(VDJ^f4-gX8OW#ARza77{ z_M*b{D!(fmj!CDO9@TfRJoWhI2&zC+HOesErK*gd{ogyfP8ZWr{+epO4nngr;ty|a~(pPKEh^lu631Js07QpY|e31&@3#xq`ABEx1 z+t+^fYxuRx!WAE)j4*~WLBiDn=_k>9QvUH99SrE84<3b~LvLKs#Z3$Ogn?&9SrW+V zC;Mi8b*Ob!EI~#62yco=TfJYM70kVEiOq4#>;I&B!W9amiVQVNTR1Hq_m}JX`X(mK zJe;Ht`MSL1O(jPknD_WoN2iDsBIJ=)0hU^NY1#$8h{;K}>Ro6``lw~L|LYvAn~Gxm zmU2GbmlN{mb4rTwji6R76qi)43Vlrg498*le86uFz=S%aWJ)|IJuSfiET52#XE+G) zJ&;&BhL_WgNoN7uu;yYa3WJk8lc&E%Ye0?Q7FLL!vpaX)6X9!&r-BV!G@9 zK5+7gXjOD2|2MS~tthSVsYGjtGgQb0Ea(LN{Nu=3n0Fs00G6$#4dJj(Qtp>xK}yC& z<98&zu?n073uTjh&qS0Z+u?Dq)0CU!q)MK>tkdGJ^x>%Rsld&??@;Qbx6|fQ^fY9{ z=?QdOvK77%`iV+5HCb%mU@E(tuY@$Ng>K_@eHD=j!Ul3W*mBQWc zncrB^1uknMqAB&uOnR}7^{FJS7O?_4(v;)2a9nD7Sn5|z)`ab)&9j`-aq>4J z)zURALL3g_LC)?*M{=6oM00-@FWKb=`ixTB=Vu-DBLtUVzeHbeg&2&_XZJ2QMm4Xk zB4e;C&+X@TP(QEfL762t8?_mLK5KKp8WiZ{667@1)eXF}!E|mK3(J&#vAr#1?jyu@ z*xX&z-A_kA>1Hue7s(NKp>3hs&O5~Z=MFZve*9Thib_;KBC6ur^t*E)O|cG`4&MJ+q>GZU3|=gMZ2LP|Ec8*dt5*0=BG+IPD)z*_<6*Y5a^qxZl!? z#2((#y#+4l)b?UGYfcNb8g|+Vy^NXDU_s##tNK=t52>G*xqAQ3ctVxa_djydNGEB0 zA5;zH?277=IvDckf@L4`}|%IzZ}Xg@3AczN>j5hxJ}U? zHN;h3z!=H34Qya=1sZN$DjVWj^=e1 zlScj3(~&b>+bWIjLi`k#@DTSi{-*;odwidkiMwN)o&^5my!oVU*u53~e%m!aa=PXF z!&^ayn~kvN1rS#zysqF5PLP)N&^n)qXt$7LA12#ECI0Mlmvi-^7+#;~-6Q-HKOn-0 z1uyX~QnQL0&9WI6CJMP&GvwwnSB4=Gw=X;VHe(T0GsfUeOhjSNx{|p-GNmxqKSPjt za>}0|L7&`zf&^JVv}+|?~j2HuLQhR0tK=lp*7YDBLSBp5Gd zn?L!mpKwWmt&#H*zw2sZez2-MJBldTea*$#724_qxs7z>{nPTOiDRZN1tP=T?slrM$bORX{8G}gealD0H zV)zFBvfT*?*9Wm{VA#Zy7Ye!%(?xe%c+y-D&mlWsfpx9~&m?wzX!O1sRd5S*>UOlL zwcCZ9of0#&R8Ib~ud_xXM1_c~CNbbT`Op2tuF%-_YG+_}_*4AUGUL-z?6T6`e?rxJ zE?v(w2rZ&rWHCWlCW+mhat3zeM)N76E=Wqp|C+EcBT{xZLU3$(#ryj>$$Z*N-7aty z-$2_!s!PIY1Pr}(Zu~F|_>bR|`O^H9Om@)Y0U^HtU6JsCiXLy>6`|am!H+H?g8oCr zuJAr1+gYOwtj8yUsosmE8_b6a=TI~2kjjCuKw`Y-K0eb2PkLy0u|;p!8kgoNELk3+ zqC5A4;@8bcBloqopw5Pe1~vmCmv=tCJHiGWlzqd4JcbLcW8XA(+}@r7%m3w`Z(Yp{ z6_UhaK_;%UyAutnU7>#__JbY1f!s#COeA?u=Rcu`qlQzYVFx`S(tRx|kL<}e%BaIb zT$v~7Nvy>cFA`=K_ap|~&bs24r$J1*#}@fE6T9r9ZGCj%dC7A)7-yl;-~hj~7qGrr z{gCOrwd~cBq~W3AJf}@u8TJOh`M{QS0m_-6ku_A9OO3e9jT*)(-&V}X^CAd#p_Y6m z+x)ea;5u+M8SKlX0PD4f)DIsHmuryMhnWy@fj|n=ivlixG!0wGFM0e4sXFt1_6;a7 zmeBYqzG0#oqlP2^{BW1QeWD5b-vroFT-PT;1OKqpxan)-3%D9>C0qTqPKBUP%wmoc(mD%`^K=$o$fmjIxh4BQ^>I>BBeY zx}AR1U~o5`?lvFHz@cmh>Y}*Von#w*gd@E3U~@F>hG)WUoj^gEfiRkXTBn*SvbA^2vLjGQ6w5&?Zy@l4MAZX!L{ z3B4E|Xj#CF7%_eShQDM%Ee(q&>wL4O^HEHAi->Dms2u~TTK}LGuB^1WeQ^g69MHf#(6iIZ@_vWuBRkM!d2o40hkkY( zb-{jC*94|XgM+6eAf+HmU{W=&iS)XKf3M4dI##pBERo|?H#6K1vsJOyUU)uAvG^R~ zbC+$uuL6#PJ9b)B^_|grzru*1Y$6`{Y`Ag!zbkN=TPo16NV)iH`O3DTWagLdkRe-& z8lm|QP!V39-G=ZGbF}Q`4{zPu#oh&44XAi(aMs1TWFu@nc?LT#%nNLmtMM^NXA9^E zc$AO?w4Q|&+kBljL(lxz+JPesb!u@d=rlwTA_~PT&ieHl7S%Ribixu1#8LzNPTS7j zVnH&`H(l}PwTUM2%C^@PqBG+hz7p|)7za52zGvUhEU}7!KZEqs$XN#*YepKD7y~6x z#eQ7P;!;xP+*PDOAiLIZ{S>Kqg%Wj%Dh;zR5aZ0#vxb+f(mc?2Ku&2d<3t0w_9-w9 z0iVeOzoKL@Leo!^bTU_J>|RxanjV;^-aKc1F_BCPQbYD7wH@HBw6$X4V`3g3-X*tr ztZHi`pPvo;*3@1pJ3E=Qm+NSeA)j5qKoXo#e+8+m)b;#XvtZJJ^v+lOGV@7G(MV{9 zYNdrqJ3A%*jYW%to$)*E<%mZP9Vzm3Zyv_Mt(Gzj@UQ{HBrqn6>lwj9-em0igUXKo z@Tb-XQRlTUCtdv7jrfLD_(okTm2;8SS+?m|VU;JNyz6u>-m`q)LfTFH&T?HI2pDDB zF5hE7GWEL_}q#u{9vz?m15T!Zbs0BzzIU`pd@k9I)%_j(_T4?((#KKh$pPi2OY9VAIw*EVb`hFZ9`u%6> zdaK6=RLv{(rGIus;OjgpmNZcArhzC{qeD_Nrm%O=n%#*tKC9d`8;BtW>*fylaU+3< zr2+7Qrq7Qo^VjfrKFwgF$e5H9`d)eljM<4<0GO)H0D(_31%h7p$roXe|E=j)0jC=Ynl*#RT zr2u{$A+ASv{JcDISI?lZD^%+BnJsq@cDc+7H*$|wa8(J z9Q%%4*i2%sI3WsaH}STQ!ao}}XWpcQfP!*+n}0YX>4}br)xh2S`SzUV;plCq#&`Y{ zO8?c_V+mNw=xUkQ;Un9=|?TVD<7;|_#49Ks);=0=f8qB5BK?QKx1 zOZVGQlXX8scZdIW+KS8E>1#9n$I|M7ORJ!Nt#CH9=v>wz8w8lOF=*XwU^tu1x{&U@AkdE}7Jt-DzZ9aqF`N-6Z3S@hoPqhHv#>9&;t)=eX74J)^+L^rSn~&@*`NiBiPkDh48(9t{{*lC*DH-h6WOdL2nTo%g%_ zu7ry{RVhpDMc^wik5;JD{f6(So>WHyb$z_cX%GMd6r|xeE1=PX0QGmbVX{PCZ^q6V zKgh$dzg82Y6!=8&IyKjEm2waJmW_64ZP*NIYqqg&Z2ZFULqv~pn)DE`pnBDiiJJ53 zftaJc55Uftn%VsUSMaNn9R@JCs8rQPyOlCMXE-r`b-AUi$S4z9JH+*?X5n_^v@mca zNLo$vEIM013k3m9fp;$y)g2Lb>UL7m`sy_YwTe`SmwH{pEvw=vcJ~edOSmptZ~P`6 zIlIS%g#fEUkZS_W6+%H+;F|U;`0K_AAQXb?ur`TA$A#l_`j<&#%xa|DMbS_YKh0Sk z>RjE*Hrn!9D>%yD)q9;O27C@V`<#fOM%neX;MX2fU7M~$r`#_U*)|)TMold^xt{NE zqY(f35eJ;kdC7^H1xdu^4`oqcnvZ#Ts5z|B9t(0gs#iy**C?Y9h z|A3^MC+->*dGO-%k0n3VsxE`TniJSJeKmNmV8eSt1Es&~`px|a=m6Lfphzy;DcL=y z0N61Ss^DWgzJ+~hs9G_~>;=e<-3_%=4al3aLyhvA-Ty6<`DEN)&Q;-7{Vk)Zm#A_+}cX}Mfs`A7K&d;6>M_;H;gM4W!MVRjI zcs?Mi7|VRK1z)%^C}<8}DnET|JBxKb)Q1N9wPI6a@Z$Ce!~LobW9qq51u11@?Dmck@6)|r<;p{AvwBK?!Q#y;;NGOOT`6!eTrAKk z1M-*o#nn*UsVr~HThXo{rzawALNnpH+panLQDl!Vq_Jl9g#E8dcV715aFNHlkn^oe zdri5_E&7YPZFsP1!YGaq#O`kmf5H8uHxZ9V-$M^zm2XePf9h4hfi1&bV#AW%*^}!N zZF5FvdmD$gJ4HQeJ_nE(aexVFl9OAS`Q;A+((mB@xru(5t$nxG=I#ujM7#%sYQx=d zmNV+B`x3-J+Gh8~R|!Hye*!TfM+*Rj#p*YIsK(D({5!_b+AkLaQg+%$0I&UoQK?^S z&l+y0nB4o!THvh+3c}`xMAME}>@~(`o4A~)uSCirF6(yE!^CY%bllFubHSsHxM9sO zqC<(OBj?^o)V0VUcn6_8XWe;*5NzsS*-TFSWWu|H9I$PIi=b+{gKqc4@Wg08d31o}&M((UZ?G3+rE08y>Qo}cFmYO!n~wEYnfg%wQ#?wxl=js}^|!5o8CM$5TrPYI z${}Ee`7+kA9mSS)ljrn{5LL9R!wCmOAi(+-M4hkhreJQ$pZW@f_!sI8hB;`uUOh_b zub>NP9ml&Q-8c-3@gj*+b8VXALu76&ft~dOb0qorVvC_ zr%3YDjv(`HyOHuEyQkkGV#gEUs-yXR#l}1o2oh5ZSIXE@%BV860;I+nNmwfb^Y%97 zPt7sFW`+Aic%EQhlnQVPjxJ0(R(0QoM`mzBkTC?dz4LLf{kxEqh-Y14l@E{Wkxwp{ zXN;vY=feQA@2O0i(VCu40=v@BmjIoG@>S;jg4xW9X*(#^(AVXnKMaMZkqvB#BxbnK zr|a3`=)R9J(lv*ef-_UyS$T(dqu{@SD9&G2z=*-hJBogu&brZXBEL7tV~lb)s4&Co z+tGd|45qDB@LRt+b$LNH2Uv>e>uN}4oeY_wAX#3RQ7Dss!CMoi0*u-y@fa_3--a-3 z^u4lI;CqEYjBE)L!t+YrLfNwSz1vV50+cMMT~broVkN~)JqZ)7^=-J^A}=OFN9ZBS zav@D4ro;fe$_xTfwu2if$2UAX()TUOO~;n7#%kObkLO5BME7Xh&x(Ni_9he#n? zR1D;Cf@#*(Ihu+m^v5c~7pnUdlQ##4sLXF#aVzySKV05xMoMO+`DgE|iy0`3JIY0` z(EMZ>ytb+s>hSi|o%7+=tUSCVo{xrVTG8eq5&W@J7-K&WAf-V4SK)exI{PWIUkc$T z2{DEcwp;&?%pe$hg}s9%3Y3btt;{NU#h%InX0JIH)PA55<6jj-82D1jyaN00spMwl zI?2yWF2jn3up8B}sm6IL0f;A|UcImGO~t(J>AII)kAVRn852H}VMP2j>zOVmSgSfJ zhl=NsmPdT4Od}1i@dLawg|FnO^A|H|%>4&Ep`JG=ZzZ*Di!U$&n(1t%fOCBw_UZ4I za%=6Lk4%oR>o_}D`ukf7;9Q)YB0gVFbZ)~zCR&}dZ~Y`H6^7hhRpt)b2HR&kthQeM z^Q`d~l0K5S328k1M z&jyq9W@f0)B$P(8xBlWFHDcd9&dmd5Zk80x;UTLy242lP>`x7t0a34zjvzdl!>X{# z*UoRw7#%POfOnm?I`5ZqR6$YfXT3i-HB1%|;ZgX+u7z++)U_tq zzucSb4ya1fO)92HUrdJ5*Y~s{iEzNq7oy9S`(?SwgPmz=J8T0#t6-RnTdY&Rlme^} z0mV;DfU9H2nC<6L3{}i)-0X}{{`TSby+_e@uM7;zF`ZC3l^JOFwc&1Irnsk;whv%X z+gN8IY7r>OunJcpz5Yfia3%bNfi;vrFD*Y4BpWbtEP&MaJ7unY1r$RD@0HVbnzcs8{Qj4wM48MQa!hQ(L zV^sNOuNCxH>$F&N7m`G7_l;=#J8tc}iJ%t$8us~{xQOea=frGvRrOsF6nBkigSP>} zE*?@ezGq#ysqC}70UWRZbP9LB5AnPkfn%VgDPn0V!>-&Tbtw)OXlJ>{(Wpm z=1pGTr2d|SQ?!@tUhvkR75EEh-eg~iOGD2n)DrW7=52ulF6p2 z1Wo{gP)~fft!vp4HR&34=^Y9XEnc=WK_!U|F^FFwnd^E&Q?7g5yjE!p1LeiFS!c;J zU%OW_&jf}8!E%U{GBzqUdnxf-etXF~2UQ~N+1+dv%3HK}aZ;S3)T6||dMni7FPD&9p55->?JFY8yU|X;eJ&B>FJe=@ zd?5=rr1^;}_c#XNx9p}O?f}IEBHN7ni02ihnF8id56dax@~jmonf;z?`|MuV9mP&h z1y2GE{Jk}ab$)XRK@1Q6R6idZQkNb>s7F1~o)piO$cFTFCvDlKt8*r^r|JQ@x66A{1UKCbjz0Q4|!T))Uu3N+tJ%F&MsYF`3O;t{$%q;^_; zL@41RqBx)IhYQk=_Co*CIyuSY8+o*;Y7(RVNPL}YG1^Z6+aiXJe^vJAB>(^~yF*dI zhIy{RI3}KoUlRS!4Fb3Y$m{W_r@1&}VxV|Q`IWeg;FsNLd;Li0_j`28sci~}+2t2` z$j}nyTTr}5ejAIWMZ%uG&5cvK>k47dR6(g*{~<1TJMOqjEVUP3UCqUxx@*<*_K8@* zn{S2a`3$2l5dK}UpXZXLS~{gz*9|@pf9)Xl=k$-Z<$8=)Dog;0CgqvGh+TEW=3K-5 zBQn@Gw<6=<5>XX0KO(sG#R)z*O!1w@pLErBdCq$t8hTOS^zh?1l7x@4PAXI({P>({(FX(UTAJ0MM@oAB8AeYM#s?Mp=*fU@~HA zC7J+WA|DhlcSaT zL&uX?Q{ux!B^!u;^)=GrU_772I1WTmVFdDaDEa5L1u7To5Knu34~ky~#hH!kYU<}$ z<tB$%K z?7D@~eTOs3-t#_w0llvl$LyuAH=dMOL9&PACDUuAGo5!lx(+FUz0Lq|xReIvbwP$Uo1y45}$o{P@p}tP%w>@<)NdY^`D(_azPsX32ebVDA zXi~!Uk0*wdQHfzGQzrfLS`5+F8Jc41Mc_?2&kbTD_)&wT_1X=r$ zf5eJEio00K=S~u21YNM3Gzg|%C@IeJk1P}oN<~ABc2UI)JcA(Kb3lNm2f~P+ev^ox z1ekCfLzi^6gNo5QI>g@QEk*u7%!0#x#h0Te3pS+5b`gkQ0Ttru^-sZkInKv7`;Z;0 z;sYM9rTKmF0h?>K>wURwfdSF&T^FGA{Qm}*BGwyDTSv(XzTMg|Q70lx#w@$^X*+md z34Q7|ln&!zUoD->mdaxAoIj2jfmZ7*;0#R9t^ooFg2_okApM|vRGD-ftsb0T6h#Gn zNA)6tS_H#dO*zNe1P)bPrr;<6cY<-20 z8wX3U`xc}3&?DNCyIB5BLim5uIhv+GM7Ro}b-t-1hYWj~kySu@<@lNgowE_Zmi7y& zZDKfcKOmB`tlC#>x05g_-`sAQC;fjZKopps|I#@iz)^2{>%tcIdw9}J+g56irt2X} z_rQq(_m&UT4QxC&K9vHob&L@Ws1gux2HjObs%l*%aSqTFolhNUgL}Nsgxz@sXs0Uc zyD(CugJ~o)9e3|8^gnelWpNIUdG{hSlRsOi0lhM(IdeyA z2YDt?SuI z(e@2wBFB{`00K<{zSl^Dyy6qyL==o$szrB6}z z#~cornj1Z|hWnAUvWUFtpfru0a4p(~&R#R0YVzMJ;xc1Fk(91F^-Rz0xxQ?!D{Odx zy_cpn-`{C-Y`SWjjZ5WsrRq9sKY$CKxfMYmp#BW9-MBR)uQs#DrR^R8PU4(H49y!P ze;{-85+^8Y5t!7~1cy=mw#rOP6Qqk~l*X{B#%w{v60ClTg(Vl3iK~BUctjNuHlwt- z{k?HnSCPi_k*}|q%;@f(NF%b|y%W>b{1Z!oX5RZG+EnhI;ICi#tI**O9CFYyLc?H9 zEgn5S34v7&Wm|23SOa|3W0CC15TKM}-vXh>MJ=DQRQ&*UEtDPtq@sN-oX5zb8d*WL zT%1L$uE;cyA0GOcuG%$l0e28YE2Q$>CYxS?XIPqYl}i?fZD-{eXZDRMCI#4}=(Nwm z{*D_B{DrV;0E!Wp@Y5sM*(pb;z|f1SOO?qf}iFem$2v6$M0Rn1A zvz+a;W>=1m^4Yfr%f2)g7x4t+HK)8AGH11#}N(Z_Pii-Z^i zzr#OqiZD?9T(S6_#({SvR3OiYpRV_AWsy(#xsqlDscQG76!YCSSOrC6QRNb*ffeXy zleWMf0{w-Y<0at@@^aF@)79oizGuhjKM@uTDT{B?n1&JY?4}dKda+1gl07*mn~=!8 zDoBo-l@&($aDeWe+}|e@Cb3)G9CYDl7jnd7F%Ii{9?#Cqps=6%o51{VHF01G<)wKo z(lVOt=cK#V_*$~G@0-q$6pP#RS5(gV|DT&!$N7O?$ahyV1bh{S(bf{T>8iYLHC z$hN8JJ@4o*T&=H`%GO}|Wx7pr^j(auDPZ`Z7q45zmK0_f-y)NU`-d73mYW}?sX!}c z*bUiGo1fU$v`|gBsJ|m;EBDhrVEZKU9|Dn|AbIvJIVhEp`3;X1lWoIvem_G-acy!8JX_f#W^PY z3nIp^{)^hLK0pUag;*vtmD-bgwh!y|CRe^eYemu5*NBG*;~efAM_kKHC#D>Ix$YZ? zQq=9*bH`JLulE+^Ty$yhU z@CBH}mlu-U2Xos&Vy(oZ93$O+X%DR8mVWYWBA`@Op)xa$WO=c^vneJQtZxz7bZynK zliU`ir`9PNv{^`^n9{oG37!B9ud~u`wQEV(;DD& zZok%KG$xBh06#xd+BNs7;Ih!s%CL<1v1JDGmkUYno7VzsAg~j|elN)v|2!a1_?gLG zOI$)kF?)d(k^pcwfz2#*Cw}Q7lOob(nKBge>dufHevrCn2R`#^GbF#&7zykSZdqKu zFYZW!0uw+`JyiSwzW*icaZ#*OW(pogohYnCX09AY>7c#u5Z$XEks59xroVz9 zH%mVQgVj*aE=Rn_$iWZ@aCpjV1<_6pD#pzY0XQBR(4bQvrW+w&Z_l&5d=L<=NH7)4 zQg17t5BROe^%Pt2ZALXU8p1k0_0Q&`+4v1Fx}S2Tn~4hcyKY$*F@_t3E(dOxBbTal zo_FJT1>51XW}P%IqqK9lv?v?IC#|P*-0t`l(pW8cU?e>UXoa?jNPVe(j@1Jt6 zwi{Kfz^Z+eb*{S(7r)uLC4Z+%A8@7R6?=YmHIgJL<7EVdwP*;a{=LP8GkU08_|;Mb zRR zVTYHEw_v^Mn-K6X_J6W^WJ#7ctn%fQH~yk9T>s7mK*H+<#R!;XnooWD)?3oAY)qvY z70=uM`BbKctAx|2n}PsPkYkj2Y^uqDBhl~-#G9W_(C#pq;{#O}5fpi7ep0~dzex_+ zT4Z^#Da0XNVwC*fKv^hAU-89HJn2I^t#U}E?TU=AwX2^Xv1qHjFPvMI*;^;DJp4&x za&dK-@MRnH-(by9!@g29-2%aIm3HnK)n^GeHYl!*JP-F#F`g=$Gy0h!M^EA6Aa4F`vc3Joq-BQvk?e{yC(ZwxPyG~vE6-G~GF#GMt>zC$Mf=T1Dm?pnhwZXf5n5IxWJilC? zTMMooM01HGz5;$+OL-YDEf{}7cF()f_ucnG$dJ$HJ5SyyMk3}3<4=&!ApS9|yx^3Q z9#?MRg$0=oN!%E-Kv2r{8N4vP2cj*Aq?t43*4zOMJ{#IuWnw* zj4-*xYV_|oVUoQu9F@psi@ef9hwbISmEL*u8-*<6&jM;gS)d{G2#3s|2_q}ZTXUeY z=jXS-7`qMoHXhAv3+w!zxJ&BaiT|4J*Yh9u=K-tkPx?$wibr{J(r7ZFtz$_3|FXv? zr$>HxIe3GWVAg>8B36k?|1up9X6X~%-rwFv$>^YQ*uR^In6tC~KbqP9E{9+I&-$bV zTNVj`TO}qh8Yx!=4H02Yor-_o!Rjt7s=GP0o57# zF(vp*tq|shzI+y3NM;2u%4-}#YwGT%PQH3tA`El01fE56lqL$!;zPP`d8r~Y^6 zh7Q>OT`RY-DSwUNdAF0HPt+gSz0*7HZ;+5+?Wcp(RBM>wbOzJe_Icu6o@5u8ks#}T zI=7bv#L7CdCSmChFs<-jO9%HpA~8>wNpt%_EuqhU6_*O&wG_ z{loA1%+PFwZSeLQ%t=C_zwTn0+@6TAHjeHxR?;Am9 zLWxGgj{DF@5axXDr7Sy*9Mez()opLi19$3^bI=aa_WXi8CRU}}{iQC>_XCHoGTv=t z%Y%5IhW9ai7WW+Pwx>UOks`m$%Ih2OLUK4`9{V!BZ-4d^>wM)lQD7wj;btVS5X#iP zu#W_B#|rY=ns0Tc#J-DM^sC3COI5?xlH;d+^E4Oh2O3yB_-Ua!_q|l!%HXB z9_?;XKL}7G@;S#!OWUy%pCOxF5D|M(+?|JDz7s<+#J*Tvd(O-!Bx2+u75p*B{`(jd#p137uI)WZzDiWDPZG;A(R~PFm~(Po(9U%5MqN(2c{)r- zo#hD2D5E0ZKA#_3?WS?t?+_8LqJ6e9&C^i+yF2ZT(~6=sR_i-?Lg-!ABf7LEn1^Fl z#L2jU|AllgUjIon^qcJIQ)`dajr}vpK-IU+xOqPH@ULS znzYqBd~y|8V@xm*+DhWr$75)Uk?MIo={6k^R@}e3xkyxM)1cPb9fL=B+2;p`bZ>r5 z$Yh7Jq*cHWrdGj-G?|=Jl}s-iELh5|9?!}6?kRS9L2H)7!2bnDb^vsuD(zG0M{nJD zkweN}V?2|kDmMZ9BLT9Dy@XYv6U=j`my$Tg=>jKww$s$jc1r)Z05KfRgtu!T*yDZL zsy=tz_`4TM_k5^3QE9MePF9Foru&Wn>#nH=DFe)Sr*aqH?TFcOUVJoK?`qYbLJTtB z$GAlcolUvag&481VvBl(5ki)P(LGtQ$xv*yl0LpR@uYv$)}KSo`nr{g-`HUgLGoe( zF3gW*Q?8W#XnDq+fbK_{t=GKck zYnYR;jACJ7tua@v2%>MORm!Na>g2g%Y3dsE8JgwKFR#~~dBcofvF*1M9@KX|SIfI} z>&fp)ic`hRn#%OYzGLo0M0Fl3#d{o~2kssB$je5J#wQtecaA8wx0Xb~pO+j5(cy+f z7zjmta}oS>N%~I@&#{V8z$BUd2G)OkO zyjI>C13@(%x+lqR10l@oLa8yC5Q5a+ZKp5TkfP_q5e@Up)(z=Fp7f8us)!=D{_aNl z7CG7$hds+7nfLD6&-}KRP>R6h0<&&;|3FMY0?Q^IQ<-|-m-d?Gr7j9K#6*2dj;3A` ztb79B;%th+L4f#=9~sP~(BJFejX{2qEHpgG4fJ@qUKwms z?K^vrHL4v#uuu*LNcpWNgYRbLp5s(J_R|4s1Tc3|!sU(=fKdB13r(s-;fh2I6@Y=F zXYngl35EB(i6TPo&$q%DO!zWJe^87j_X`x+;!t50TboGluk1up$#L%QzEGU&XraWb z`p?o)J&s#`pX4#HNwAE3F37%tul#jGYV+h>zSW`r@H0c-cyh`5_lqgaVYn4={%>hS zOBHU(Au?eD_C>;>q0`q9k3`R7D3+nnHXRH^lPRA%r}&&L$5je8PxZ zJ;JN28(#b25K&Pn$$HH3@z5`me$2*1<`$=!#p54EmBlraFgXe_7@$U@FaLYgiB(+p zoxD&GY?`n;Rs-lqX^}R`3ixCXz#!nkv8JVk-KFX!muHhzH210$@Erm6+4_R4${>vvTWNg@!fYl zFLR+!f9bbUz{p5>BoL1$bQ+=K&=8eSyTC&3+EsV@o#GjW_r z!Km!Vr*P&j@WWfkPw4yT2UN3Wib5F&w-@(u$|twN)EV?`OAx1@3L2>%Hb}k`BTD~F zp2!#v(YVme9%0BNU#U(o$!r`CN>^v;%nT1?CzCBaRi>>^gaZWagG#$EMwfF>0>2Nt z8L}2o(u;E-VXj3HzvU+O0!BZidE_e^eDw_#)ncW~9pVOV@Uq2JgNvICTb!V!aT2DG zBAa8H3nC4^EX>~bVJKs@VW;j}@OA0AJ7T4&poHfPw9XQ}a7shl6T=xlj_4&>b`xWz zm-eLUOrI1jr9iSmGU%Jk^ai$0&eGULrmW)c5iN=oj;NgZpKbh~5BiL)xqR5Lb?#W$ z4P(%V2xJthIQmJ%;+8oUwqS|{D~uw^fzit}DTXb69zl0=Yjx5ed}AVEjIv>Mn%Ve0 zVh0vOcAZ%IhY5AJx7*d{{h^UiX$%ibd#5j5!Z81Jd*{>T(0z=k7hcjGnnXv zr4;SINPscBcx&N;Y95o?y2$dK@Q$RA1n2TlH9f6#V4zphY?F zbr3Ip4Nem$bf=R&tpBJ-P^(#yjom79B2)gV(agN@P`8mPQdoo^+{9NjtFJdyc-!~m zockO%$iYmDImk4Y#oDz${2te&?Gd_;K&))^QTzx~zqvEw@Fq@l-fKg}1maM8pQdc| zR%!{V;Qq%ELX-yU{uGyQ5q_Im#oS#H7i#=?7Uqx6+s;QC537$(VfDocS(bfnS=?~) z)KhY&<}x}@%`kz%^LY+zy{iVo!r#`lb@b3 z36Pl{iDaP73qA$OOy{{pNDpY_8G;Y%e2kudbqZ^4)_wo&;#HzZ=bb49KZR{@>o`fR zc6^@H*`^15qnr9@;>y2qlsUh$>OpT73efUcly`GB@RVufc7XQe{u$@B=J-D;Ko7(A z2k*P$6mT^<9|JGdzDd?rRHe`ZCPYf$1T8}OZTA_PTT#!+)z@BTnQ18+^r!Fx73i`z zr#;8|PKa69!7s};+h@-+TZHbe7Eg|^de>)L-hN17VTjPW{~Zb=c(cx3hB08pIv1ih z^C9T;kV9$C-N~kEZ!s-E%eEx4^jb^E{UX}43f;=d!^1N5?$UeiJf~jfzV72|jrDdV za$bL>39Qa%83L2<1ne?G)D(wE9PVE&=+1uB6P|bnhxE(I`_Usy`}ykli{ImqRv6u= zAIf`s-@;aWn7(`#F$Y@Wt~T65+cUM8EZ(pyBIWYk72DV^mk-|ZvSRz) zqGB*=H~tL#miSq1?>dak^~N9%BW>tnvtgj(=f(1S4_>EGwyyN|wtO~th@LCk2V+(_ z9EU#3+a`5qEqoS`9xYOl?HJM{I zwGZo*PRyhP?TI)al0n99IrJv+y%*g$UkPGimbA;mV!6vjAyZ&GkB6lCIAdU1!# z=u(C?o%ju2OmY-Iu95~=TbiZR0Xc_DH?uE3P*<@ww!F3F zUyY_&_T5unFBGfhCWI7kZtsHf(dZhkm7bm_?#-WQ_`BqNLKJv$y!H*3X@ApU->qaoM( zb4ZYj%l*jj$z#Xmd%Fw4sj6PCn7@wYL|`HG_-)-md9=k=tn@1Zjc3cP`=?#im;FSZ z+PAxwsa`4$GwWpAV?17m&@htWN#gs)XG}^#U3u%vR_8-RZ2FhK@3~f*&1RNrKx@-D zMuZ7%-4DktT4|20N_P_>gnO7mE4IF-Yy@2=Ww+N#+lK}h^(RX`bd+>m0t-TU*6o?& zQABc;4n0f*xD&1C)+7_RSs1ru9e29VCYNIx&5SmaM$VIwSmr`Y44@N>-GW-sMjOPT z=6Xik+ZUx|5a*xu2S>L^<#tp1aiic4bY5p>mUUnWH=UhrcE8n7*dRR<_^F-WMKt+( z>BOK_f^LiA+4CEFuZ{h&JavMlW17SBENZx@{mPZg(C;smksc6cH4;(yLzvWDN+k^>F<^3#yp@_V8 zBk>=u%-IY4paNZ1icq|9*4PlZI$r-UyovIQ=bFsi712N2v9a2`bAPij#A&h5u7E4y;*P zSGS&Pb%0Ew+6%#kfUe~Et=@!9*E;$4TSFIw;tSUOmJ&$2GDwk{9UY=3-_i(ZSZg{L-+n%;hQV(XQK zIM~)B6xr4l5D2)23oWmD?Vm6I*0RJ4RD}%Buk_rC2)$brnrHv*DE+IzC3_Doqy0?E z4lsC;t4)9&NL*SNvK!I4xq!9C(OW$p30j-bHfLySk(u9^K7%EDzw5v$tkPY(u+ur< zMdt9}^ZUr|dU|2M+bVE);;J=zyIA73+_>)yPB8S$ui6%S@~#n2u{EwtH_{?)pQqt4 zHtHJehahEn&@d+5?KaGB5+0kDv#jK2ziv|L4<1p`(w>YMo! zS3=hlYTeeUYqN`H?votgRLreyJP@m7dJgP~Isi}XRf%Us5~GxR;2VopTeK?+{EVVe z0`D?P_B+Nhrg0|_sEHOm@<>E<78Ajh_doj%X}NVJ^sS+~e1X;XGDkk{sZ5TbwV|~K z^n>191s&MuKyHPE1<%{Wny#F!=8ld_i`$A0oGzD2#BQ$Z&w_pY?sMP{f@gczL_WU{ zJg5}86H9DJ-^H+RU@9&&aG3dxth`=e=(I)6rO_8efSTZ~ZPK zx1Enz;88$tIzN9t=W=QGsdIJxdK8RRO0DMMUSde%&7QjVMgqPkgJ}*NVv(dxhf=kT$SrfOI5wy zU^0T~N*6<-^tN8|`nIuTS>(YfX#*qdO$nF5V!?%C|F@I-*5+$;>u(LSkUMRgzKYdm zv(#nkA$wO?O_rzB*7S`#6+BJKvS=Z-K^0Uxi1f@DT1EErszg9`3vrSVmR#gY%~^z-ySq zf{Ai(7XCjZU1dNUZL`HG&>+Q~;!@nb6n7{^ic{QOin|ndcXuf6!L7Jkg1cLAZr<@cygf z8bY3+W->7bAaR}N(bF#y6phkbqu~}sEw}L;=wTSPrL`;Q7i?yl3SZO*V8S1l=a8)# z_c?n_!2b6JhEsxs=KompZ~?KT1dU=xBQkk-e$@_QT`HcjV070uZMmNi8@k?gv%)@; zVk&)jkrb8yS=F7Z4xmCv- zo*iJ3Fz!22dN;4u+=GAaRyc#$r(`seHp-yk<4{cME0+nXu&sOCzgPIBlyBEu^uLnp z@P1A|XaH7yZ2}r5V?s@nM?CmmK#MY5-5Q9J38=D5CSNrz{9o$;P0Hi}4P)r;k=V~u z%Sbac-*^Z8A0pNA$?_$oEd{N}1mU0`O?o0{fU3ogg?qgYLw`Ll+Ty)i(26mO&$@za znG%20B{vqHuQ>6I*;s&0q;5KzWIA~~OxE}1e^gG+^?H04YDDx5SHi%lN0lJQhd2F8 zG$|nXMU<<5ZVQD-fur^ku^NHmzN$uu=(UZkYE8!o!d)QQfm||0lt)VbCKL1KeGuh1 z^JX{&h$S0fJ}BN^4?&TUKlBkWDJ^#2?_J#*(lJ=rnQHcTn{9%z{%- zGQ6UIaQ@6R)n)cqcnR|R1K+TRrYFpE31i(ViS3zd`$}_v*Ooq?lSX_4JME_@yzoh` z{quN-2V3b_5!UqZpoN4;QPWhYd|wSMo+1_dnGVS0$-JEOT4+7bsLBIBj#-1o9OUw0 zdE?y97Uxhnroe3qVy4B6WCu#h^Q?jD7k!v`ELFQIETU`yAm z1$9E`Lb9JHsNmeIs>I`KKTUAaJ3i41@0(bt8aZxtP@QS%nT5k8yaPnnCk0SLvfK;>v{$Q5~Kv}UMlcvb5e+$WNv!gGUCeA zHZw-q=@nFW>P90QNq}oI94J4V}!YVEjxXc48TlHa}%a?sM8*V;>b>gl0}{YEE^fBEdryX-hT<{5r@&zHl?L*&fq zI!%vYQ}%c3Yy(d#n@p&^8|5bw4PNvi(d(^R2*7~sHs!V46Y*SB1XfLrtIXQn$R`<~ zY-ba?!MtrC_^-7Q7=Ei?^4;ZWr-!!B5@NUz)}-}V(XxeO)3xW{&rNcG@;kG@8cb>G;C!WF_1pf+yw+p9hCM{;pMFAkJa(a@ zp*a(e_lo~4C1(~o23c1(L98ziF1t~RUVOXrk%x=c#=}!JW7Xrlj+wz*m{3&CHbBDl z4TN|=Yka}O31j%03V2X#$?=7uU-=~!$(kqNB}@6k2=rXM_C@QlyG2^FJY|b_$<)Xu zAGThWk%Yx^Sa6C=OfelWnE4Mnm;@T_+g`SB7HTr)&5Sy}EfCVD5yCe~#Z+5oT)jn_ zI`B)9tj>kngKB#j@R1hXGrO((bd<@FHYe0LFB+*83@~Gtsd1a%mjlT~s{28(RV++@ zfSHif!QN`nPO6AhY`QremHS^FjBxR7;1QQp}G^6drY`0C%b%m#|lvygz&{up~BD`xr^L$;1@1io(pxo zSlWI-Q|s{ULH0Pi|MY&LJTm`lh)|Y*lMXK$mvs3~kLr&mcmLk)=8iNvrE?XT@xc)K z5kIW|%FWKzXvabseM5f!WQ@cpR}~dU%QjNjJ`p%7Hg8$;FNMJ|9{@_+Jq!hUK z$?;Wk4F;yI(B)wV+(p&2`#Hv4JR6=#q(;(LjBAol6c;fOz{V|jb#INkTvtL!hH-fy z2-$pikAT#|t)_}YR}a$NqFAajUU%63WfzyoQ09B#-y*=f&;19YF`MO{(cq?wMZ=k< z0IEsoD3ai*ToE{+pq|C5rEFRqv||=Ex16udP>J%1qeq&?Zw(X$r)1vG+BA5&DKQT&0N)lDp;HZBjUM$7#qPg%>!Q0x!L>Gu zBZp(cT+_x&M>+=C90k+i4Dz4iIoXM%-I@yy$B{@FIr{f5HwRB5uZE{e`Y_G1zjt>4 zYC+w|1=7J3*bXm`}6lC*eTf4^%FuYjJvjpO%0|zs!w{3Ge(EwZ_*lZ0+xkYG7_i?P>RA z_MKz)owr~1YI-W7Hvr8I=vV}wLY~pZACF`?D$bCuAA|MI=uZn?b6CQa5*Hl~hthll zhWD>TFqLA4!g*0&9ur*>3-SrEL*NhWkYSeHLdjM*an|hL>Og{$iZC2+?2Lv*3ACc9 zs&gNgy{MkR)pQ8; zLq)2pX*@hcYQqvn8Ue7Om^W`X4PCvN>XK&f*2c~*Z@{Gau*!@)^>K<1KvBDYz2(l^ zaLt^yRb+Hv?T=yd14F{qZ@~HT;lZtUeWtK}nqPxYDR}|qtV`~9ty!e|KU%%PvwM{q zu6J-T6HCU&Njbx>g|SxoH+s&oLRqnDsnm!z=BteaS&bG{B>Q<)S@$R$9o#Mu=M=2nJc}M5IOhi zXTT;(#%VsZxq9N(cLsaj-GV@ulZN>hmVm{)iS$f|InHu#dRO{0WFgSs&|;noULrPp z-`#r4(!IaA1JEAPE4>PEB}9N>H>dr4GRm&^%)1f(Ag>0UwQTE<7j+y z${#^Tskb^>X)52`ashk=d&3*RwGyV4*e8`XTFnzUmt+iG-$R#_?b&QSULR5C^yW%@%gPi8iuzlm7P!XX}Q(?5#!^k8~fkX zE3uI?NC{4KXoi^Kc)it6$c^p}+E7|OI(vJ%9)Ak_Q<&RsXSLqKl3aiG5mx#)ad%*F z_)!iPIGdQr&cCv@88bJ=&LalM{vIm5$lc2f2PUz)a1}5t#p*^6o@LhCGo0iT9#UWr zev-i;YY88&VU-HI(CBl)%>Z+CyKsfPINWWH0a^mbP?GDB^}fY#T!fyw z&I$MaG!DCLYxi=a{CruuMAx@?K2+%k!T<_#9F|BA-wL`9FI!3pJ=~l4J;sYpgb$_` z?@Zy_OCM=)W}KXP+>j$Xo?sh4LlAvN?vt@36F3CDY^HXE4(6#l@BNTBT|DZ1B`<;V zPplnMTK_}^Uo5|ZeYDV`HIP2Q**y;v{1a|E$(Vwg?+X`Ijp%QxEOe|6o{VouB}t61 zkRDxs*#v}{#}K3en)1t+++3(}y5@U($!gd4^4|TaE)2F&Dwo!q?9pk$F4MJ$`D}q^98Zvr+?-#~CkYL^}($G+sN#$kT`_ zoZ&ozBy%4EkJfn79j=c?-+HPIcDkknoZZ!vR#epO#66)k;z{1`dc_Gwa^#iclEgS^ zdD5&zqDS8pQgd?E>zO2d7;R{K`*V5;8h@EEH} z0%3ITW4#rUg``b(rHyvEsK|tVW9y=0+}m>@B4|orTt00SzhTLjZe#?h*&i_)Uz-na zw%({+U8d{2K2FAz6hIfa+~DSvFY3~@S!$Reo?i*WHHm~=#2XW8l(q)E+XIyxaa&9} zku;{(Lt4Vuf%8W2{B9f3S!M&-_R9B5##E(*^_PELgnnPSk*r&Vwt-?W-W@SDLsd&X z)+zPdyO%|YuKJLrjk=4_nbXL_R5_zPwpM?IlVRDftNoEp5OAUA6QU$row@VA1`cTH zCav9hb1{hpMLON`xw|qS_iNAS0>NlI4t5%v=4%65xKcz(t=n|=+0BK;mBQvKV{9}^ zq-=rZH%)KfM-e*tI6qUUdKzUBMG*0mEuRDQYimwEd{F4}t;0pu?jl{`_oM-3X9in< zU3x69tg-PEAu=8{BlJQ|D- z6o_5`7+rv#_$Gq-6ct@$g7GcBnu>}L@(#~Gk_71dtbv@wQ$LE+A_Y}(&1H208r@_N zl|x}B2|zN7mo=iA`r$`p{;KbgfpkHh2Rf?T1j+|F*YBoQd4xMY9~$xQK*sTpu`^;p z-?N`VqoN#geJ`$senbYR{KDiTlx<`J?>yIuqcUxr$W)*-7;#Yz*FU#{J0DTM_*q2x zwUca;inRIo3F<#q;qeN&VRF`jVsvtZ!qp}BMZ$hT_G_0+3q2ljPCc#VzN%EHnI#Yz zdk54M%+`cSizkuLDaFEWW>318txcQ6e(-L}AMA-fGVaf{BDn8eNy}?<~I~| zc<(m1U>kD(82RTLew9=Z+?1%!G!c$^j1oypVX@(FN( z-?Q;p!iS4{?vTvaR@eDl=pW`&Wa#JS{|hM)2c6ljgJfaF%9W60YcfFR{rN0G^KIHD zFVzR(YHeZxrw1KGf{PzKc}2lsS053_Uw=jEXI+9(7IT;d&;b z7&bk05Rda$i_#7~P(EVxd-I}j**rUCpLxssQHs+A7m|>3!9+{+3jMLf1n%&gJH}+5 zyDapj(Y%zM6-fr@S#%)^zy>-x@e6BRn_4uHiDp)eN%_P0GF=92hu+qj8w1+(?)u`P z{oKG7YGrcFq~k^koiH$-hk?Dbr-KZ~My=Rigl0yWbdN1Q_~us@e8JjL024j8mK;y! z%qG*?{ufzP=#pprWh0AZxdZJi^59prZ;Z>cOaT$VE*WRmnf3dL0-di9pSd1Olnfq$ zBQ*@(ShK{*ttV|)!ecDZL@awMCT!9j;}lHxI3L-IXR2cO6ak7U`rBn&cJ7uXfl;@` zWS^arRX(_Uw3U01`6fU_jx*Jd+_8^@P%J=WZWXt4lE!dUye{Y~`yFZrbFv4;Ptx}4 zsfg$93s_96-jKe5753IT9drp8neoN#<}j~)R8-9a3lx~vw?Q~Rn4-jQ44bR?L*n}# z=CQ&2ciwqx$x26U=f#0}viZtg%*e*E7$Cbp@@hLrdd^oaSGw!3Vn)*@GuSk)i@o~C zR}BAe5m2v*np~@3OOKYd-FVbc1NK``Gkz6gD95gy9Dw$^W-FNfpi?<6O6pKHi30PF zk7dzM8Q+!z@uk|C2wcFKfVF3xS!ICDXJwUn!5P9CWfZG%9IJCafd+lNPZPy@wmAuU zTdBgSFRolEcvFL%|J}ca0F8-NoXF4`%aJehJbC?7=|yxpX(Wvp99VWmTleWQjjWa& zu>Vx^#Iiin)`nlQeMCBJYje^V)%tX6LhWLG%co1JQDxn`;0KNc zi~6;C3Dk(~xI{QSE2TE+Og1^=xsZZ!MG<+Tnp2t|ORuu|S?yG)cDI}agx~bPRx;mm z^Zx$(c^9W^mztaQfvjYF%%<+%>CA1lF29n_Od~G1IvbTnEJtrpdaGAe;w#0xF=;o) zH&I?5i_MRhv|5@^-NVgvE)MczM_8|71kg~CKY=0lI|r=Cev!$`L6%FWk4A>yz+_5B z4a{Vpjo!$bw{9TOs4B?I(Vzjo`m>X|f-FEkzEs+shUFr^M2O;n$h`K(V#p>zL3q z$6s)GIcq%#y)bN^|9cv(Cb-w$b9mO{HisVyPl=!{TuJ3mG!XXNm^N0qqwPWIqe0r{ ztyu*Kb0F%4Z zpzJ)$Od$kX2`YXj&TXFp(F*5QZxN77n1Y0JWTiQ9a2e-9|r^PZ5D{M z539(tLTcz8!=j4zZ?Y(9M`#bS&Pu;a-+%iZocgk?enJ<68B+Y{+hbzlz5u(B{CYxy zBIQZ$-h($r1KAqD-WCUiVJnA|&(84WN5#!!$=9w&a9{i=+;F*-vb%FTEpnnG{-!h& zDXZ~$O|x`9b>)TW*S%S75jyFdJSn5F$X-WZkLHtcDzyw3FN2Ghsa%$TMO{gUY<+(2 z&j#-^$%g(Qo~7v~%Wr2v-)JcmPd37a>})G0cj7A44NY7wd2(S4#hiB0tVozeJIcM| zKmFjK24{q<4#&d4C>U@7ZzbV(+2te@IVn7Szx+~C3|{AJ4Eq*84H>ild{Yywx{1DW zFPV2nhrPO(IjwIJP&tayGKlD2iS|p{gSUtys*}9loUXsQvU=D>1PXlG5_d%-?i2*a z71sDFT{kqfUR*1@Z2eUrsW}2RciCz+O++n7nl(Ph96|HWO{{=n`y+6LXu*e<#<{?| zK|Gu@Tk*sdMEMTlY7Jh2Z$cjMz|K)5Y64sft`1Ly1D*Ps%6w{oA%I5}!B=}J%JoD? z<%fxwS|U7NBwBSfi%amMVJ5lBpThaH8C9*Gd_+qBgajUiGh;2!Y!lSQSB%sCxhaIH z+8JuuaBbc7KhBhC?X$$@+3+bBnbX~%kIzP#?_@45?OtE#F~^`7?<#v&ZA}RwDWZ8Y zX$=yxF2D!E#Vto9Q@Ep!H5^M!6ELpo*p(_)xbvgzhNjA{t{sb%+Fkn_LDzf+ru;)_ zqJg0T!SgcV7e-4bLo2xlVg0UZB#XEgjU$5} zCBE*@`p)@-b|O+-ArQ4hN;>w)`-CQ-;(VBuq~L?idpV-uQcGUZ$VatA>@z(4RUJKt z;FY(Le`sB+MWB{~%w^l~zg%p|!UDICDitO&<1+UwrAhDgk{o(b_|Zz%(qBH zxkty?b%NzQ7|QS3h`#8t0;ae!JG4pA<9z!{VXc7>i7de=q8f6DSxF)#FUUhlGRyks z>ya`uy&gDFToSL5hyP_Ms3Fx-Jvrf-tvdBtB|qUagY=SdQ%Anr`eXU>r7G!vH)0D7 z!bH#d1u;&gJ$-Scroww0=MrA#o}VgvhLsq*Vg+byU2yJe>|rG?`VfBm%rnr+l0MzP zKELn-+pUx5^Gye<(-vX{D0DlhsX~T`#9Q1EV0NMyZ9XOt=vqmz^0O8*Z2?T&Y1f4_3mrd&n%lhl{r>Ym8Th!>Sk&(mK0A4+giIUE~ zMRUWk`D*+shA7#i) zy?`(U;JK(W>AV>*dY=ap($N74MC+{S>SO=P3rN=0bQ&+it2wp#?@Y-I;n%7^l-?9a zPmAFM01SS^t39ek$&AKSg*9(%_o=z3{_2N0f%`nKz1dZkT;-oUKb-Wh$VG7=p!~d$ zQM!asAe&-%g4n#*HJyq3?t?nji%@e}ZxwPov%!3|COVnJDvf3*ew+&A}9d6rI1utuG_ zgI@;3pj%7kGlf(waC@bGM1F~4GS6SN*8j8Ie;H-)DrWL1KU!x115hygV*h3quH-74 zGgy4ozP*@0wE6}K_{%s+%Uh1}`r0KtQmxlQg<>@i2<@>iWlO%K`?IAJdm~w%OvuUU z8)VGwPdt~9edlLlY`Wlc#D=vEmC8LCc~Xzd8lOl+=^F6PdylWiM;A4KifP$rUU~{% zMnh3U$%PO1mmcg2v~M4LfUrpI`IC3?U__R_XB?wW zM>FdysIa^$7OeI8CwhAem_w^*_#%{{J}nOC$+NjbvTX0$SAUF4q*5}$&nOs^H=EB? z8a6fcMta2)?>ez**{#=+R!}}YQ3*-$jGE_Ym-l(}$_QG#2jL})VC|3bILvMRxBCqu zKPZvH_aP52l0{ZriM&wJQ%w`(wW3bP(DB_lCVJ}uZSo2VTVJ?zgV(d7ySk~*zP?Mz z3wz(#*+7`>$w&)KJ&v0X zI2a1+(d(IC8~ELYRcm?fDXqU<$ckZSLCh$Il`8UM1&URP!Z8Y!W@OKk2YJoHbUR@V z+cme5@`cwkQP>$fFZ2kAycEzCKCn~D+`hi1(n~T)cWTGbb`X30>v|u`pN!?k?8yNB zpgZ#rZrr&8-laP<@y)y_F(2o!1Rdi2w-ZbXNH5KE%Ca_$pQ2{ajT*c5+ln+xb3j|L zZ$!I+aRkyeV%V8VpRg(9N2C9Up(wzDcK@X$E503htIkDRtZi_o3=P{rYn7$nKk(hP=3A09wRF_aq;9H}CL8Z9abn@`W^fYW_#lgM|nZy3a2UPZR@=$VuZO&)Jt_cOm&n51D-|Nd=nb7psX9sOy>!qmfZ zPk|sPiO{p~mkgX56*Zby60{Xq5+*FHbQto2k4qo|Hfe330xfU{%Shpe(JaLRT~PhT zDay?9ojA6_om%R_@T~wZ(!vYen8ZMomoZ4Vo zbR;HxnS|`vPuIo8U6qPVI1Gob93P$<1aNG*@@s}-`jI>DwzClRNPV66hrQs{Dl;9= z2an3C@AY>?TYx{LJBXisrm&!(Ek;dNz##?1F-et>t2KeW`DC?nbt%U2vu{sB{~Q&H zFJBRSM70^+cA(JCO0}II+&##mU^mUz^HWLgIrE9!QqyJwUF_yG4`&vsUDbOYPXTcL zYi#xpb#hm^t-6!;$F3Lq8Fz_)5u z3|BQ$&j#F)+MV+7vx&&~PeO|n3lhFPB+NYzy9Z>I#5HDg>$K*vS6+X#XcFxH1z#oA zWRB7PwEdPmqb|e$O(Ls$7&yi!o)2}_1>;`8UrNd!pD2#1oNpi5h*0|b_(pt$_4je3 z>%8Zve@0&%2u9bI%mkEzD~IMT)Qwk4n|!+coMo4|lUL>SDb(VOu65+rHm{>+gkB$7 z^Tf*A&5-Z!XM$$a#jWhy0LYAjFLMt{X?XDB^FZ=-9R*>gCuW@qhsT;WXjvVVD5;{l z0Yc~k=U-rJ_Cp=`6rz&6Jwan!{x%oMl3GYttF>{l?Sz7I2&xfN@vG1!aYpexSU=G*z zZ4a=34-=uZl16ExvB#*!nw^#sq1!|F4jqZM!3d{F?6ZWT(+M8l3|qTpU15xutdGoO zkGVC5gEgV~r2DMgN`L2{niqO+%wRz|pj9Vgt8v>p*N#+4jLUW5;<}$3aia+9^)4hz zkJXU3FHE-memVTJ7q4qNCSu~@e=BP`;gNkt%hjBlRKm1u4Cxz@7#fQ4vUVm)HJe)i z$;eg9hnYjXXTcs4AtO52V>!%btv*W=opXsnNW~e)r+ggYXBg#J2^>}%OMuU%)zx~c z81>{+d{7`v*WjGcIS4+PCSRH^{e;AVz9pZ3fSY}JZ6cmJ((m!{>NMpy3e%L@?1d=&&e00 z+nC6?0MwAIFU8b)pY;yla7)R;!8v;umfoi}8tviVb%T-q5qvGpYm0q$rzcte6}pCx z0DW7&PXqsyPsUVq^~SoE?24@4yBd!?JBrM|&x3CV7P>9B6iD|RJrMjsE9;aDs;kFR zyoc6&FFLgA+AI&H2+-$QMcCAzT?p?L!2TO*ym|K#R8aol@fRF~i|`7iSwG&c4bHsH z`gYYDnM>*sQFq4COB3_BwqEE9nhXJ)t3VqJQ|y1dNGOA5uUBV|06vp(Wa2zJZY8ZGOk;#{@aQ2XVF9w%q?nXw2P6@p7LHXGefx9@*6VckLA7 zM-9~SOG{k$BmqWEUJhV+Q=ju2LgyWd8=nOO16FCh274lK-<$GIWCB9tV(kT?!sys( z`@U76uCqoaG33?JUJJYcIv)b?;`f7?f3|O&_bT$@KWpBeFtz+B{n_vr|NZ^Fkn)iGQ~@fsfK*!n^HXvFtp_cQo00Ig|7|{n+UMk2{8WmOIN)Oj z9c#hMhtM2A0#YS$i|<^M!~>dsm}=xe~rd2bf)7~9PqmDYm>q1Bov z(s~hpCF-XzUj(hvtA<~g2gFV7mZnr9-rWaXUp)N}&RGU0(0I4^Lx}5tn;OQ%vg%g? z$-qI645=l?y8{-ww>@p)kmR1+ZrzyIE}`k$p~dsQ9p3ZM2A!3?Mz6~U#g3URK|5)x zmLw?-s~V5&@+HWmFcpducmncdfRR@|sT1msR7Dw2Y3av>FuJg0=korJB)^LdIE9($JbEVa5iWiX;gFa(;yYUW&16J_!8I`?eWwi z7&GRk+{y@_zSFs9=j@71#_z=hdLM_idFvztzcj!pLtFZbcB&FyKVx%rz^(R@g+}^W z^p#8REeqIm;M}#)ji$_kgS@8PL|ysEJ919P)MT2UD8+;0pw9n z0>ftRYY4RLGlBouee}JyM{D0}f`EhlH$6u>OtSsh)_*i!DLy#Ayib4IYeM!ZYy~Xjt|sAMz3@g!zzf|_29@jd>+@lJ4j2VSqXDfa zw+Y={`5nbiw+@zBOp007nbp)?=gU`S1|0|zgATIiDI93+yl?#TaZ;-NDj=*@>F7VB zg+|YYw+s1#8a%%w!v$`kDWc!P2*^#51L(jgm3IKTAs4$7J z*bnD6^X~LDxw7$x@_ePmhH0uL6Pn+b@JT~CMjl_F0uY$rf;hQ(@F#!jh4q|U^&TW^ zd|9*ag;H{dGs8T|Rg7)je%(=pT@I4clx_UJMFziYmtGHMrv?sbR>HvCv1$P4&O?3e zzHm{wI{a|S+iGI;Bcs_cflo^KUAOA!`2iKSKwe@kmYMIRVmXApmVLktM~{YNhY<>TMO*h#Oe2tgM*gdv?AB0uN_GRbncMTrk{^e%7H2P++CA${e zU^vzh5&a(AeahP9D;~Bl{V~yJ=c@wh0e2l{4jqn-y~G8@Izb^my8>8jX{pbwdO_=4 z0FF=UWi}3mXXt1(4<{m7987p^~+D4+dvLER=%lS8ue-l7YmubmYb zDOw%6aMa7Rrnn3~1+IS~O>l9tschQG>_4-p^9-V;+mqV*6TwcPGzf6Zx)GyMBEt%K z@!*!rJbl<#>44UmOT;3)%rNv~cp2IhfRT}Emut$EM$W%4=S$2^D_V@*Lh*jrGbZU| zd2}oX&eC4ADb6v~4dBc0R+#ermcZ#2S`MYnB-VUyLBfybHPo(X`MCV!EPHIpQWi8# zb9;8xu?u(pyA1|Izmt@dhZpqwnhpKjW-yq)US?`8zosv=HdS*cyf%n}T@RZLxBA*2 z_u}Cb!EkArPima29K<*;m)garwf+a3+l0%BPeTdK z7o_D@>y9#|k{&&W6FHk&Zy{UBdhnkWwYs6DRlYMa1wIx$Dmp2uuC+gw>=BF#8TerZ z>Wa(Vi(YooSBwfdNUgLwS>p&mw$Z>a8uC1j6Ecv`NNxDcV3CK92AxGVQD>buFkIzMdwW? zcK;Y%$W0%MmQ`+`EM3ZD9T=cUTlG^PkZO;!J-TE{ntWE_>a_kK2TR7^@u5UOFrz}? z^7SMjfIeLT*aq3#dhH=Qyn5Z)AN+N500Pd*FX0<6@-q;4d4I9N!@qy-7F<_2ioKvrM_%WiQxB@}YldHAbswgSm z6Oty&fV+|r^xCSii&DrrT~2iSdRd|IbH*?*&%7CzD{6Z1_$6-E7__oui-7LG-0!Vg zy9!ZKayfRu;hK)?Z4DvsMGYJlISD9>g;9RB?ln{DJ-jklfq4?8K`DL|asAQ>HmB1= zkc5X<#Os=E%&nQyPw5MkL1a-@XEbqb_5L4rsYAf}HnOhu3@%h7orkMo#i{cDy+IIv_H7{R!^EW??y&T)(>E68l^ABMUU4=%8 z#e~J;&}81iqxchMzz}cc%!Ap~NGqdS&}|r;yYS5E-}V$YFXd*i|0;{qLK<#HDl?pji6YO@Wq<`d%*n zY{sdps07_FCHgr=#RC;CQ|!mTK@7>68Qc%@&eNT=T7DXCEIK!abI68d;SYZcXM;?N z-+WSCI`6CCV9Kzu>+D*KYH>$S;dc@PlE+347uh|o4bfFxy%d<}HWc7N#Rdy|8dc~V z%(ilWkd(8NgsLo0?Q-xU-Q9HRYFDthz)4CB{qm7&n1v#q%)t?|R$qkq3733+qQ%m0 zuTh;n+{;WoGw3r7_bAB+eRqPfPsfYYyL0=|EKpy&5)ihCbiv+b_5Gun`A451MJs50 zx&C;{C*)Mk8{T-dCS`-_#5zn}c3JADczY1()FCP4rHMk)opS&d=0epTm6?XZ`G!v@ zD9bN;R2Aqy>tE`{Td&{ZvCh_hx9o0awp6Ni*OKmujj>cK#em-%C@3PHRT!Wnu7ZC6 zh^z$+FC2^)hMTnR9GO{Kh4BwI5;k^T(I#cZW-4mpBZhNx<}U{PH7-pV6`Enw`R5qLwaxlw1)<0>K{Jj~}S_ z6xp}4p0D!&^-!Zo#J-yIvvM?SK&e(luhP|sNgf}2^-CQ^$Z}EC09pr~^}Q1#s%4XQ z_*#6kPK38k(S&+}3V+DR9-Qw@Y5Yj~Ja>X=*`%y{!vWNR!uo&vjF4TxYc3`Pb)ME- zx2L`c9QHYdnO=YeA}T;go;; zERO4=`y?a&W6d{HOnE@be$jvRsx2z&@;wd0gWm%Ev1`Y^zGbD^J(Gk%De~G@0-L7} z#{I)hP#4Xt)FX^v`ZTr}K-QsD3Q7p#hBn~xG!U!^nrE=eCgFW+pg3Kifu%72#Z~j$ zjYPk-nV>*=AjY0JsD_LnJFl-A%HS*8NU#sqXXv*^)>gGXAu1;=j@zn;pJ_Eqg6&}? zT0Qc{-#xt2G?+He9So3m7liq+?6){$c|dKw>#@@+AyhvcWp8bZIaqV;C6`46!5g4x zPd&g+q$XI?xbUy+2kfc->w1dJ9YcmGhwT3vIKj?io)$k`EYmEh#BvuymGWVyF+A2U zO`F4xzq+)r7(Rkeut$@-fH&ITDg5lR|9rRzgH3(FzROWHYa7aUk)4YhbA4~Dq&N`H zOLzNnh6;<4S_-8Ntym`+ITVs_Gufqko5)pzt47pf-CWRkj zA`bq?2cJNeGS-cm6eY!g!Eu63WtHu#^dkXO+Yd7TW~Bi}%o)R#prgBRi)1Us@EYVs z$+IX)yM#-fS(egnf$bHBTI-+uebnd?bRB;p>Fe>JXoIaFi{*S z$w9}+n7x8PX)bDPtc0JGLMYXP?=v`;no7rRT@5I>khY2ncA@mK!CAkC_ z?`JS+u_0u#dAD1JW9pQNJZkhAQeyuPE<(w$ySx^HLW7|YKWsIR2i7l~mbD{iZ% zN3C1=s=^wk^A@J956vVmS0Qlk@S1*}Nw|!JHD2W~bh_!VdM%dPffN)_DH3lN2W-!y z+SI05nKP6+;3h{{-tWp9Gn^p4Pk@iiv<_7xh- zu0PM;qIrw4Jgo0*~r45x4{gH}dmd&URgnM$Z=SpVMF z-Bz}@bil2d^oshK>T@lAk?W5wE!u$sI}cmCTsYYo0XjF)(_@HQufJJ4>;-QjNPfAP zjkb4dSThe^MnbfMNXn|yYxdzTTyh_X}^6IRRK^@I7M_(h;eA^;c`0fEJlG&aA6fZ6dtVj0@%g+ zw{owwQAge=R+1=|TpZnc@P6>m_y*{^j4H+uB%lK?#_hsBaoj`tZO0k}sy;m!LV`B@ zqJ*-QoVp4Q^!USh>^_L-&YM9Dp&w%hqnwp={p zxCKy^effnCn_aLp+TAjVlf!IP7!ixitn*3w?xooZkjrY%{o6)iyHq2R6clSKyGW07 zELcCv39e>jt5}-K{G0u@5^I?U)f*Piz*de5&Qj2-0~Estm3+Nhkj8!Z$!q<} zDRvp%pejHMdq+whbwEGc{75+#(JyldDxf|1a`5U&E55 z;87>#D);nj%+$5GU&tS#=Kl3CgVLdBVr@}l9MR-3utiY<#<{0LN-qrU zQ3_YYVEKJA(VYuOa}%9ufoeGBTr7 zl3UoW=aqhbKX;695R{~UZUc^W^`*YP)%QGTWYd3$qloF#5*`v&2fkS}|H)OQ%DOX4GGd^bF#VarLSk(9# z_;Q90R312ouikTf+e@lmqVmBHbj`h#XJ9FNhX62o{E=rt^8VL;UMeq**_EwEmjs7$|#gF_3Ed zI8>OnEA;7!0P&W6Sbw8taV*Xm=Xk?m1ovC+95VR#Hd|Kri(si<^2&)9)qfHg-l$=* zIM`7_B5L!GdMiwtA7n=6DU9OAAu=r;uP0KSJMzalyo2FusmjM!m^L-en`q#%M5mCC z#`^=I>mZev5}A=*xNS}@Sf;uV>$1sFhZzjxOnjIh+Jqd)4&-PY65cHGe#dfRzhn-( z&NWYVM{=g$-6AK;zNWBWPmTZguMUwsPm#O?HzWt?F}eS_9-4|S6n6Im#4s;GJ;5;x-Rgw?5)bY7f;>=s*2Z^qI{{Yqz}inTQLh zKAs$0;Y1k0m?aFf^p=pg9iP)Rmo{!9)qx6u-QTa=7->u`*xpa`b1DCC{>}N{c543k zv+{68cHyIhiIV;EcJux$PMTnAD7A=V%KNONx5~3FTzpr%L)zTj?8yqPMGrqc-faq+ zWcI3Q)SMaP@an|k*{3J;$?ar_nJC`tUOro)OX=N>OD)1dTW83uc53r$+{I^Zw>HB) zaP!^ob&Trrt-vikw=E+BD|Q{}5o?!rJzF+s`!2rC!lp9>RyS!d9aUq$XM3rw?7r6d zfpSt-Wg4bNF~9n2{`~&FG{e|3aN}}O(ds6BNxQdK zRAPR4{0qG2ylCIv@6(wJ%1WL(TE8rx86f{O#_GP$y5^bUO{~ParT)kiJ(80B0Cev>J(f%ws`N|2IV}_U>FEkfis|7p?=e%*Zu&FB-duf2*2mNsF1g#e$F8MWYu$?d@Z(dbde#$(^pHfy zO=A0H3v-rmzIYct{m&d=QVed{e2i~qSecJXiJZ^EN%Q4bOZoWSkd;$8rghlw$i4Yb zp6^#{p84m`^dufKX;@#oKkZj+xsFFq-%_%&ByPTRQvReS!W3lI02P4DoT`)5YXi)R@H z6)qu>cRRwkJObL~Of`J#>;jlQ>*NX(Aa`j34@}Q6Fqq>5J8=AP6YR)yB_$~_;60mA zaB)KdLPr}b;`nb99q8?uz@yX`B_dr=$;-{Q400Sh&rKiy^pNf>k%@J{i4wP$ht3L^7y(b6Ncb_o;z;ZiCPzJK^3WW?TOI{n@EMD5yy3+2#jygEASb}G#~rGEMJ#IIK*RPIk=-FZ>^ zdKOUSXLH}?eZ??;eK{>!nBX;myTekM^W^q)WnkCx-}T$qXR{UWhkH;w&*9|x(pEO$ ziaiE~>dNB_-Uz^)^4E%|BGeAJRF&bu#+~eYjN!( zdEydI8D~U8Z{|>16yM0mzKq3@fx+UnEb~1{nE0=+5(Nof=2saQ3Z~pnxP8^^fhb0%lsvi;r44AJXl$CEq5&gL+P|OM@^WO zT(=vTgBQIxtG(1CgtsCcZsGOhfJyWDKTU%Rgf~2dUgH8v+06VtJJ}QU5whtA8a2Zg zAKtCV%N`F8+8=BpzMJy0TUX!YWC%#lW>1CQ&(P8F;<_U+6}xQb_&mLLgZ!q2AA7U@ zzXon{Y~Ga&ckyYl!h}jzfsCpk-! zmAw+C?eLK4l9lw4Zdu>@@E$uXN?ysjQ&MhpcZ5HEy{;_NGO9j{k>QArr^}dcyes=9;wEVRJzp81wwjTF zcYXNbt5?b=tlWPeUU(g!=Xla^YDuZpqGeJH488fGha;cA_-qWEW1Y$0&%{uXlXqor zfME)A7Nj1B?x=s)8=-Un`7AuAp*0D-5gpUXO@geCy;JWi source/_notebooks/explain_llm_logprobs.rst + +sed -i '' 's/``eli5.explain_prediction``/:func:`eli5.explain_prediction`/g' \ + source/_notebooks/explain_llm_logprobs.rst +sed -i '' 's/\/docs\/source//g' \ + source/_notebooks/explain_llm_logprobs.rst diff --git a/notebooks/explain_llm_logprobs.ipynb b/notebooks/explain_llm_logprobs.ipynb new file mode 100644 index 00000000..d697eadd --- /dev/null +++ b/notebooks/explain_llm_logprobs.ipynb @@ -0,0 +1,649 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "7c2ae8b9-9525-4b24-884e-3dc3bd9b8b8a", + "metadata": {}, + "source": [ + "# Visualize Token Probabilities and Model Confidence in LLM Predictions\n", + "\n", + "In this tutorial we show how ``eli5.explain_prediction`` can be used to visualize LLM predictions,\n", + "highlighting tokens proportionally to the log probability. In many cases, this can help to see where\n", + "LLM is uncertain about its predictions:\n", + "\n", + "![LLM token probabilities visualized with eli5.explain_prediction](../docs/source/static/llm-explain-logprobs.png)\n", + "\n", + "To follow this tutorial you need the ``openai`` library installed and working." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "71c34fb4-503a-40ea-b166-63164ae22ca2", + "metadata": {}, + "outputs": [], + "source": [ + "import eli5\n", + "import openai\n", + "\n", + "client = openai.OpenAI()" + ] + }, + { + "cell_type": "markdown", + "id": "9a4cacc8-b0e1-45bf-ba48-ebe352da9ead", + "metadata": {}, + "source": [ + "First let's define our task: we'll be extracting specific product properties from a free-form product description:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "355d48c3-e24a-4731-a7bb-5aface7d969c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "json\n", + "{\n", + " \"materials\": [\"metal\"],\n", + " \"type\": \"table lamp\",\n", + " \"color\": \"silky matte grey\",\n", + " \"price\": 150.00,\n", + " \"summary\": \"Stay is a flexible and elegant table lamp designed by Maria Berntsen.\"\n", + "}\n", + "\n" + ] + } + ], + "source": [ + "product_description = \"\"\"\\\n", + "Stay is designed by Danish Maria Berntsen and is functional and beautiful lighting\n", + "in one and the same design. Stay has several nice features, where the lamp's flexible\n", + "swivel mechanism makes it possible to direct the light and create optimal lighting\n", + "conditions. Furthermore, the switch is discreetly placed behind the lamp head,\n", + "making it easy and convenient to turn the light on and off. Thus, Stay combines great\n", + "flexibility and elegant appearance in a highly convincing way.\n", + "\n", + "Stay table lamp is highly functional, as the arm and head of the lamp can be adjusted\n", + "according to wishes and needs, which make it ideal as task lighting in the office.\n", + "Furthermore, the silky matte grey metal support the modern and minimalistic expression.\\\n", + "\"\"\"\n", + "prompt = f\"\"\"\\\n", + "Product description:\n", + "{product_description.strip()}\n", + "\n", + "Extract the following properties from product description:\n", + "'materials' (list of strings),\n", + "'type' (string, e.g. 'LED'),\n", + "'color' (string),\n", + "'price' (non-zero float with 2 decimal places),\n", + "'summary' (string, a very short summary).\n", + "Respond with JSON dict with above keys. Always make sure to return the price or it's estimate.\n", + "\"\"\"\n", + "completion = client.chat.completions.create(\n", + " messages=[{\"role\": \"user\", \"content\": prompt}],\n", + " model=\"gpt-4o\",\n", + " logprobs=True,\n", + " temperature=0,\n", + ")\n", + "print(completion.choices[0].message.content.strip('`'))" + ] + }, + { + "cell_type": "markdown", + "id": "b3f870b8-8b15-4fd5-a5ee-6836c3e12d76", + "metadata": {}, + "source": [ + "If you examine the prompt and description above, you can see that not all attributes are equally clear:\n", + "- \"material\" is quite clear and explicitly mentioned\n", + "- \"color\" is also clear, but its mention is more ambiguous\n", + "- \"type\" is underspecified, we don't extractly specify what we want,\n", + "- \"summary\" is quite clear, but there is no single correct summary,\n", + "- \"price\" is not present at all, but we ask a model to make a guess.\n", + "\n", + "You may have noticed that above we included ``logprobs=True,`` in a call to the model,\n", + "this allows us to get a log-probability for each token, and then we can visualize them with ``eli5.explain_prediction``:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "17bf32f3-cedb-4aa6-9998-e6a4bdd57407", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + "

```json\n", + "{\n", + " "materials": ["metal"],\n", + " "type": "table lamp",\n", + " "color": "silky matte grey",\n", + " "price": 150.00,\n", + " "summary": "Stay is a flexible and elegant table lamp designed by Maria Berntsen."\n", + "}\n", + "```

\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + "\n" + ], + "text/plain": [ + "Explanation(estimator='llm_logprobs', description=None, error=None, method=None, is_regression=False, targets=[TargetExplanation(target=Choice(finish_reason='stop', index=0, logprobs=ChoiceLogprobs(content=[ChatCompletionTokenLogprob(token='```', bytes=[96, 96, 96], logprob=-1.2233183042553719e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token='json', bytes=[106, 115, 111, 110], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='\\n', bytes=[10], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='{\\n', bytes=[123, 10], logprob=-3.7697225252486533e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32, 32, 32], logprob=-0.31333035230636597, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='materials', bytes=[109, 97, 116, 101, 114, 105, 97, 108, 115], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='\":', bytes=[34, 58], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' [\"', bytes=[32, 91, 34], logprob=-0.018151583150029182, top_logprobs=[]), ChatCompletionTokenLogprob(token='metal', bytes=[109, 101, 116, 97, 108], logprob=-0.001557355048134923, top_logprobs=[]), ChatCompletionTokenLogprob(token='\"],\\n', bytes=[34, 93, 44, 10], logprob=-0.0001307142956648022, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32, 32, 32], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='type', bytes=[116, 121, 112, 101], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='\":', bytes=[34, 58], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=-0.07268553972244263, top_logprobs=[]), ChatCompletionTokenLogprob(token='table', bytes=[116, 97, 98, 108, 101], logprob=-0.8554064631462097, top_logprobs=[]), ChatCompletionTokenLogprob(token=' lamp', bytes=[32, 108, 97, 109, 112], logprob=-0.00018125296628568321, top_logprobs=[]), ChatCompletionTokenLogprob(token='\",\\n', bytes=[34, 44, 10], logprob=-4.644463479053229e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32, 32, 32], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='color', bytes=[99, 111, 108, 111, 114], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='\":', bytes=[34, 58], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='sil', bytes=[115, 105, 108], logprob=-0.16051970422267914, top_logprobs=[]), ChatCompletionTokenLogprob(token='ky', bytes=[107, 121], logprob=-2.4391956685576588e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token=' matte', bytes=[32, 109, 97, 116, 116, 101], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token=' grey', bytes=[32, 103, 114, 101, 121], logprob=-1.6240566083070007e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token='\",\\n', bytes=[34, 44, 10], logprob=-1.1472419600977446e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32, 32, 32], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='price', bytes=[112, 114, 105, 99, 101], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='\":', bytes=[34, 58], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32], logprob=-1.3856492842023727e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token='150', bytes=[49, 53, 48], logprob=-1.2531815767288208, top_logprobs=[]), ChatCompletionTokenLogprob(token='.', bytes=[46], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='00', bytes=[48, 48], logprob=-9.019237768370658e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token=',\\n', bytes=[44, 10], logprob=-0.005233763717114925, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32, 32, 32], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='summary', bytes=[115, 117, 109, 109, 97, 114, 121], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='\":', bytes=[34, 58], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='Stay', bytes=[83, 116, 97, 121], logprob=-0.9748213291168213, top_logprobs=[]), ChatCompletionTokenLogprob(token=' is', bytes=[32, 105, 115], logprob=-0.4495566189289093, top_logprobs=[]), ChatCompletionTokenLogprob(token=' a', bytes=[32, 97], logprob=-0.0009273029863834381, top_logprobs=[]), ChatCompletionTokenLogprob(token=' flexible', bytes=[32, 102, 108, 101, 120, 105, 98, 108, 101], logprob=-0.6902044415473938, top_logprobs=[]), ChatCompletionTokenLogprob(token=' and', bytes=[32, 97, 110, 100], logprob=-0.04642750322818756, top_logprobs=[]), ChatCompletionTokenLogprob(token=' elegant', bytes=[32, 101, 108, 101, 103, 97, 110, 116], logprob=-0.2864536941051483, top_logprobs=[]), ChatCompletionTokenLogprob(token=' table', bytes=[32, 116, 97, 98, 108, 101], logprob=-0.07355178147554398, top_logprobs=[]), ChatCompletionTokenLogprob(token=' lamp', bytes=[32, 108, 97, 109, 112], logprob=-2.45848218582978e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token=' designed', bytes=[32, 100, 101, 115, 105, 103, 110, 101, 100], logprob=-0.1320674866437912, top_logprobs=[]), ChatCompletionTokenLogprob(token=' by', bytes=[32, 98, 121], logprob=-0.05520902946591377, top_logprobs=[]), ChatCompletionTokenLogprob(token=' Maria', bytes=[32, 77, 97, 114, 105, 97], logprob=-0.01817324012517929, top_logprobs=[]), ChatCompletionTokenLogprob(token=' Ber', bytes=[32, 66, 101, 114], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='nt', bytes=[110, 116], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='sen', bytes=[115, 101, 110], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='.\"\\n', bytes=[46, 34, 10], logprob=-0.1874067187309265, top_logprobs=[]), ChatCompletionTokenLogprob(token='}\\n', bytes=[125, 10], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='```', bytes=[96, 96, 96], logprob=-9.615255839889869e-05, top_logprobs=[])], refusal=None), message=ChatCompletionMessage(content='```json\\n{\\n \"materials\": [\"metal\"],\\n \"type\": \"table lamp\",\\n \"color\": \"silky matte grey\",\\n \"price\": 150.00,\\n \"summary\": \"Stay is a flexible and elegant table lamp designed by Maria Berntsen.\"\\n}\\n```', refusal=None, role='assistant', annotations=[], audio=None, function_call=None, tool_calls=None)), feature_weights=None, proba=None, score=None, weighted_spans=WeightedSpans(docs_weighted_spans=[DocWeightedSpans(document='```json\\n{\\n \"materials\": [\"metal\"],\\n \"type\": \"table lamp\",\\n \"color\": \"silky matte grey\",\\n \"price\": 150.00,\\n \"summary\": \"Stay is a flexible and elegant table lamp designed by Maria Berntsen.\"\\n}\\n```', spans=[('0-```', [(0, 3)], 0.9999877668917825), ('3-json', [(3, 7)], 1.0), ('7-\\n', [(7, 8)], 1.0), ('8-{\\n', [(8, 10)], 0.9999962302845802), ('10- ', [(10, 13)], 0.7310083823709606), ('13- \"', [(13, 15)], 1.0), ('15-materials', [(15, 24)], 0.9999998063873693), ('24-\":', [(24, 26)], 1.0), ('26- [\"', [(26, 29)], 0.9820121645783885), ('29-metal', [(29, 34)], 0.99844385699996), ('34-\"],\\n', [(34, 38)], 0.9998692942470765), ('38- ', [(38, 41)], 1.0), ('41- \"', [(41, 43)], 1.0), ('43-type', [(43, 47)], 0.9999998063873693), ('47-\":', [(47, 49)], 1.0), ('49- \"', [(49, 51)], 0.929893198527551), ('51-table', [(51, 56)], 0.42511036426464216), ('56- lamp', [(56, 61)], 0.9998187634590409), ('61-\",\\n', [(61, 64)], 0.9999535564437448), ('64- ', [(64, 67)], 1.0), ('67- \"', [(67, 69)], 1.0), ('69-color', [(69, 74)], 1.0), ('74-\":', [(74, 76)], 1.0), ('76- \"', [(76, 78)], 1.0), ('78-sil', [(78, 81)], 0.851701041299637), ('81-ky', [(81, 83)], 0.9999756083407958), ('83- matte', [(83, 89)], 0.9999998063873693), ('89- grey', [(89, 94)], 0.9999983759447105), ('94-\",\\n', [(94, 97)], 0.9999988527586979), ('97- ', [(97, 100)], 1.0), ('100- \"', [(100, 102)], 1.0), ('102-price', [(102, 107)], 1.0), ('107-\":', [(107, 109)], 1.0), ('109- ', [(109, 110)], 0.9999986143516758), ('110-150', [(110, 113)], 0.2855947083916427), ('113-.', [(113, 114)], 1.0), ('114-00', [(114, 116)], 0.9999098116895265), ('116-,\\n', [(116, 118)], 0.9947799085613173), ('118- ', [(118, 121)], 1.0), ('121- \"', [(121, 123)], 1.0), ('123-summary', [(123, 130)], 1.0), ('130-\":', [(130, 132)], 1.0), ('132- \"', [(132, 134)], 0.9999998063873693), ('134-Stay', [(134, 138)], 0.3772597528750643), ('138- is', [(138, 141)], 0.6379109265584916), ('141- a', [(141, 143)], 0.9990731268261651), ('143- flexible', [(143, 152)], 0.5014735365596692), ('152- and', [(152, 156)], 0.9546337659276504), ('156- elegant', [(156, 164)], 0.7509218498342461), ('164- table', [(164, 170)], 0.9290880349965437), ('170- lamp', [(170, 175)], 0.9999975415208362), ('175- designed', [(175, 184)], 0.8762818557623782), ('184- by', [(184, 187)], 0.9462873253446092), ('187- Maria', [(187, 193)], 0.9819908973956354), ('193- Ber', [(193, 197)], 0.9999998063873693), ('197-nt', [(197, 199)], 1.0), ('199-sen', [(199, 202)], 1.0), ('202-.\"\\n', [(202, 205)], 0.8291064546756102), ('205-}\\n', [(205, 207)], 1.0), ('207-```', [(207, 210)], 0.9999038520641101)], preserve_density=False, with_probabilities=True, vec_name=None)], other=None), heatmap=None)], feature_importances=None, decision_tree=None, highlight_spaces=None, transition_features=None, image=None)" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "eli5.explain_prediction(completion)" + ] + }, + { + "cell_type": "markdown", + "id": "45ed083f-a81a-40fe-8321-c9d5189c4aa4", + "metadata": {}, + "source": [ + "We can clearly see that the model is very confident in the material -- if you hover over the prediction, you can see a probability for each token -- and less confident about type and color. The confidence in price is a lot lower, while summary, being a longer piece of text, is harder to interpret -- we can see that some words follow more obviously.\n", + "\n", + "We can also obtain the same result by passing ``client`` and ``prompt`` to ``eli5.explain_prediction``,\n", + "in this case it would call the client, and we can pass extra keyword arguments -- here we'll pass ``n=2`` to obtain two different predictions, and would leave temperature at default." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "868651b1-0998-4d92-a029-5259cb11e046", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + " \n", + "\n", + "\n", + "

```json\n", + "{\n", + " "materials": ["metal"],\n", + " "type": "",\n", + " "color": "silky matte grey",\n", + " "price": 199.99,\n", + " "summary": "Stay table lamp with flexible swivel mechanism for optimal task lighting."\n", + "}\n", + "```

\n", + "\n", + " \n", + " \n", + "\n", + "\n", + "

```json\n", + "{\n", + " "materials": ["metal"],\n", + " "type": "table lamp",\n", + " "color": "silky matte grey",\n", + " "price": 165.00,\n", + " "summary": "A functional and elegant adjustable table lamp designed by Maria Berntsen."\n", + "}\n", + "```

\n", + "\n", + " \n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + "\n" + ], + "text/plain": [ + "Explanation(estimator='llm_logprobs', description=None, error=None, method=None, is_regression=False, targets=[TargetExplanation(target=Choice(finish_reason='stop', index=0, logprobs=ChoiceLogprobs(content=[ChatCompletionTokenLogprob(token='```', bytes=[96, 96, 96], logprob=-1.2233183042553719e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token='json', bytes=[106, 115, 111, 110], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='\\n', bytes=[10], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='{\\n', bytes=[123, 10], logprob=-1.1472419600977446e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32, 32, 32], logprob=-0.1269671618938446, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='materials', bytes=[109, 97, 116, 101, 114, 105, 97, 108, 115], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='\":', bytes=[34, 58], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' [\"', bytes=[32, 91, 34], logprob=-0.02975349873304367, top_logprobs=[]), ChatCompletionTokenLogprob(token='metal', bytes=[109, 101, 116, 97, 108], logprob=-0.0019255406223237514, top_logprobs=[]), ChatCompletionTokenLogprob(token='\"],\\n', bytes=[34, 93, 44, 10], logprob=-0.00012535012501757592, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32, 32, 32], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='type', bytes=[116, 121, 112, 101], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='\":', bytes=[34, 58], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"\",\\n', bytes=[32, 34, 34, 44, 10], logprob=-3.1719813346862793, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32, 32, 32], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='color', bytes=[99, 111, 108, 111, 114], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='\":', bytes=[34, 58], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='sil', bytes=[115, 105, 108], logprob=-0.0868551954627037, top_logprobs=[]), ChatCompletionTokenLogprob(token='ky', bytes=[107, 121], logprob=-1.9027791495318525e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token=' matte', bytes=[32, 109, 97, 116, 116, 101], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token=' grey', bytes=[32, 103, 114, 101, 121], logprob=-1.1472419600977446e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token='\",\\n', bytes=[34, 44, 10], logprob=-1.8624639324116288e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32, 32, 32], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='price', bytes=[112, 114, 105, 99, 101], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='\":', bytes=[34, 58], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32], logprob=-2.0339031834737398e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token='199', bytes=[49, 57, 57], logprob=-2.1854772567749023, top_logprobs=[]), ChatCompletionTokenLogprob(token='.', bytes=[46], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='99', bytes=[57, 57], logprob=-0.021091928705573082, top_logprobs=[]), ChatCompletionTokenLogprob(token=',\\n', bytes=[44, 10], logprob=-0.003600071184337139, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32, 32, 32], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='summary', bytes=[115, 117, 109, 109, 97, 114, 121], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='\":', bytes=[34, 58], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='Stay', bytes=[83, 116, 97, 121], logprob=-0.9872180819511414, top_logprobs=[]), ChatCompletionTokenLogprob(token=' table', bytes=[32, 116, 97, 98, 108, 101], logprob=-0.49120932817459106, top_logprobs=[]), ChatCompletionTokenLogprob(token=' lamp', bytes=[32, 108, 97, 109, 112], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' with', bytes=[32, 119, 105, 116, 104], logprob=-0.7496315240859985, top_logprobs=[]), ChatCompletionTokenLogprob(token=' flexible', bytes=[32, 102, 108, 101, 120, 105, 98, 108, 101], logprob=-0.8411523699760437, top_logprobs=[]), ChatCompletionTokenLogprob(token=' swivel', bytes=[32, 115, 119, 105, 118, 101, 108], logprob=-0.07119860500097275, top_logprobs=[]), ChatCompletionTokenLogprob(token=' mechanism', bytes=[32, 109, 101, 99, 104, 97, 110, 105, 115, 109], logprob=-0.06603597849607468, top_logprobs=[]), ChatCompletionTokenLogprob(token=' for', bytes=[32, 102, 111, 114], logprob=-0.7179251313209534, top_logprobs=[]), ChatCompletionTokenLogprob(token=' optimal', bytes=[32, 111, 112, 116, 105, 109, 97, 108], logprob=-0.27914053201675415, top_logprobs=[]), ChatCompletionTokenLogprob(token=' task', bytes=[32, 116, 97, 115, 107], logprob=-0.5918124318122864, top_logprobs=[]), ChatCompletionTokenLogprob(token=' lighting', bytes=[32, 108, 105, 103, 104, 116, 105, 110, 103], logprob=-5.512236498361744e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='.\"\\n', bytes=[46, 34, 10], logprob=-0.019795112311840057, top_logprobs=[]), ChatCompletionTokenLogprob(token='}\\n', bytes=[125, 10], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='```', bytes=[96, 96, 96], logprob=-7.493430894101039e-05, top_logprobs=[])], refusal=None), message=ChatCompletionMessage(content='```json\\n{\\n \"materials\": [\"metal\"],\\n \"type\": \"\",\\n \"color\": \"silky matte grey\",\\n \"price\": 199.99,\\n \"summary\": \"Stay table lamp with flexible swivel mechanism for optimal task lighting.\"\\n}\\n```', refusal=None, role='assistant', annotations=[], audio=None, function_call=None, tool_calls=None)), feature_weights=None, proba=None, score=None, weighted_spans=WeightedSpans(docs_weighted_spans=[DocWeightedSpans(document='```json\\n{\\n \"materials\": [\"metal\"],\\n \"type\": \"\",\\n \"color\": \"silky matte grey\",\\n \"price\": 199.99,\\n \"summary\": \"Stay table lamp with flexible swivel mechanism for optimal task lighting.\"\\n}\\n```', spans=[('0-```', [(0, 3)], 0.9999877668917825), ('3-json', [(3, 7)], 1.0), ('7-\\n', [(7, 8)], 1.0), ('8-{\\n', [(8, 10)], 0.9999988527586979), ('10- ', [(10, 13)], 0.8807625946978632), ('13- \"', [(13, 15)], 1.0), ('15-materials', [(15, 24)], 0.9999998063873693), ('24-\":', [(24, 26)], 1.0), ('26- [\"', [(26, 29)], 0.9706847790879929), ('29-metal', [(29, 34)], 0.998076312041703), ('34-\"],\\n', [(34, 38)], 0.999874657730981), ('38- ', [(38, 41)], 1.0), ('41- \"', [(41, 43)], 1.0), ('43-type', [(43, 47)], 0.9999998063873693), ('47-\":', [(47, 49)], 1.0), ('49- \"\",\\n', [(49, 54)], 0.04192045711003856), ('54- ', [(54, 57)], 1.0), ('57- \"', [(57, 59)], 1.0), ('59-color', [(59, 64)], 1.0), ('64-\":', [(64, 66)], 1.0), ('66- \"', [(66, 68)], 1.0), ('68-sil', [(68, 71)], 0.9168098442469044), ('71-ky', [(71, 73)], 0.999980972389532), ('73- matte', [(73, 79)], 0.9999998063873693), ('79- grey', [(79, 84)], 0.9999988527586979), ('84-\",\\n', [(84, 87)], 0.999998137537802), ('87- ', [(87, 90)], 0.9999998063873693), ('90- \"', [(90, 92)], 1.0), ('92-price', [(92, 97)], 1.0), ('97-\":', [(97, 99)], 1.0), ('99- ', [(99, 100)], 0.9999796611750019), ('100-199', [(100, 103)], 0.11242406570159309), ('103-.', [(103, 104)], 0.9999998063873693), ('104-99', [(104, 106)], 0.9791289503750816), ('106-,\\n', [(106, 108)], 0.9964064013024616), ('108- ', [(108, 111)], 1.0), ('111- \"', [(111, 113)], 1.0), ('113-summary', [(113, 120)], 1.0), ('120-\":', [(120, 122)], 1.0), ('122- \"', [(122, 124)], 0.9999998063873693), ('124-Stay', [(124, 128)], 0.37261182608821525), ('128- table', [(128, 134)], 0.611885975620183), ('134- lamp', [(134, 139)], 1.0), ('139- with', [(139, 144)], 0.4725406405098848), ('144- flexible', [(144, 153)], 0.4312133197198732), ('153- swivel', [(153, 160)], 0.9312769175066504), ('160- mechanism', [(160, 170)], 0.9360971843168736), ('170- for', [(170, 174)], 0.4877632514490544), ('174- optimal', [(174, 182)], 0.7564335926068906), ('182- task', [(182, 187)], 0.5533235142368016), ('187- lighting', [(187, 196)], 0.999999448776502), ('196-.\"\\n', [(196, 199)], 0.9803995245221345), ('199-}\\n', [(199, 201)], 1.0), ('201-```', [(201, 204)], 0.9999250684985642)], preserve_density=False, with_probabilities=True, vec_name=None)], other=None), heatmap=None), TargetExplanation(target=Choice(finish_reason='stop', index=1, logprobs=ChoiceLogprobs(content=[ChatCompletionTokenLogprob(token='```', bytes=[96, 96, 96], logprob=-1.723973582556937e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token='json', bytes=[106, 115, 111, 110], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='\\n', bytes=[10], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='{\\n', bytes=[123, 10], logprob=-1.5048530030981055e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32, 32, 32], logprob=-0.061997827142477036, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='materials', bytes=[109, 97, 116, 101, 114, 105, 97, 108, 115], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='\":', bytes=[34, 58], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' [\"', bytes=[32, 91, 34], logprob=-0.01815205067396164, top_logprobs=[]), ChatCompletionTokenLogprob(token='metal', bytes=[109, 101, 116, 97, 108], logprob=-0.0026272619143128395, top_logprobs=[]), ChatCompletionTokenLogprob(token='\"],\\n', bytes=[34, 93, 44, 10], logprob=-0.00012070851516909897, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32, 32, 32], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='type', bytes=[116, 121, 112, 101], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='\":', bytes=[34, 58], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=-0.06543421000242233, top_logprobs=[]), ChatCompletionTokenLogprob(token='table', bytes=[116, 97, 98, 108, 101], logprob=-0.763924777507782, top_logprobs=[]), ChatCompletionTokenLogprob(token=' lamp', bytes=[32, 108, 97, 109, 112], logprob=-0.000204851632588543, top_logprobs=[]), ChatCompletionTokenLogprob(token='\",\\n', bytes=[34, 44, 10], logprob=-2.2723104848410003e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32, 32, 32], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='color', bytes=[99, 111, 108, 111, 114], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='\":', bytes=[34, 58], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='sil', bytes=[115, 105, 108], logprob=-0.11303260177373886, top_logprobs=[]), ChatCompletionTokenLogprob(token='ky', bytes=[107, 121], logprob=-2.7610454708337784e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token=' matte', bytes=[32, 109, 97, 116, 116, 101], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token=' grey', bytes=[32, 103, 114, 101, 121], logprob=-1.8624639324116288e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token='\",\\n', bytes=[34, 44, 10], logprob=-9.088346359931165e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32, 32, 32], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='price', bytes=[112, 114, 105, 99, 101], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='\":', bytes=[34, 58], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32], logprob=-2.339278580620885e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token='165', bytes=[49, 54, 53], logprob=-9999.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='.', bytes=[46], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='00', bytes=[48, 48], logprob=-0.06308919191360474, top_logprobs=[]), ChatCompletionTokenLogprob(token=',\\n', bytes=[44, 10], logprob=-0.0031777136027812958, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32, 32, 32], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='summary', bytes=[115, 117, 109, 109, 97, 114, 121], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='\":', bytes=[34, 58], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=-3.128163257315464e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='A', bytes=[65], logprob=-2.0843987464904785, top_logprobs=[]), ChatCompletionTokenLogprob(token=' functional', bytes=[32, 102, 117, 110, 99, 116, 105, 111, 110, 97, 108], logprob=-1.0358246564865112, top_logprobs=[]), ChatCompletionTokenLogprob(token=' and', bytes=[32, 97, 110, 100], logprob=-0.005551286041736603, top_logprobs=[]), ChatCompletionTokenLogprob(token=' elegant', bytes=[32, 101, 108, 101, 103, 97, 110, 116], logprob=-0.38757890462875366, top_logprobs=[]), ChatCompletionTokenLogprob(token=' adjustable', bytes=[32, 97, 100, 106, 117, 115, 116, 97, 98, 108, 101], logprob=-2.9871745109558105, top_logprobs=[]), ChatCompletionTokenLogprob(token=' table', bytes=[32, 116, 97, 98, 108, 101], logprob=-0.00891046691685915, top_logprobs=[]), ChatCompletionTokenLogprob(token=' lamp', bytes=[32, 108, 97, 109, 112], logprob=-6.704273118884885e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token=' designed', bytes=[32, 100, 101, 115, 105, 103, 110, 101, 100], logprob=-0.5383039116859436, top_logprobs=[]), ChatCompletionTokenLogprob(token=' by', bytes=[32, 98, 121], logprob=-0.018782243132591248, top_logprobs=[]), ChatCompletionTokenLogprob(token=' Maria', bytes=[32, 77, 97, 114, 105, 97], logprob=-0.0040874239057302475, top_logprobs=[]), ChatCompletionTokenLogprob(token=' Ber', bytes=[32, 66, 101, 114], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='nt', bytes=[110, 116], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='sen', bytes=[115, 101, 110], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='.\"\\n', bytes=[46, 34, 10], logprob=-0.014438370242714882, top_logprobs=[]), ChatCompletionTokenLogprob(token='}\\n', bytes=[125, 10], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='```', bytes=[96, 96, 96], logprob=-2.7610454708337784e-05, top_logprobs=[])], refusal=None), message=ChatCompletionMessage(content='```json\\n{\\n \"materials\": [\"metal\"],\\n \"type\": \"table lamp\",\\n \"color\": \"silky matte grey\",\\n \"price\": 165.00,\\n \"summary\": \"A functional and elegant adjustable table lamp designed by Maria Berntsen.\"\\n}\\n```', refusal=None, role='assistant', annotations=[], audio=None, function_call=None, tool_calls=None)), feature_weights=None, proba=None, score=None, weighted_spans=WeightedSpans(docs_weighted_spans=[DocWeightedSpans(document='```json\\n{\\n \"materials\": [\"metal\"],\\n \"type\": \"table lamp\",\\n \"color\": \"silky matte grey\",\\n \"price\": 165.00,\\n \"summary\": \"A functional and elegant adjustable table lamp designed by Maria Berntsen.\"\\n}\\n```', spans=[('0-```', [(0, 3)], 0.9999827604127778), ('3-json', [(3, 7)], 1.0), ('7-\\n', [(7, 8)], 1.0), ('8-{\\n', [(8, 10)], 0.9999984951481292), ('10- ', [(10, 13)], 0.9398849290249089), ('13- \"', [(13, 15)], 1.0), ('15-materials', [(15, 24)], 0.9999998063873693), ('24-\":', [(24, 26)], 1.0), ('26- [\"', [(26, 29)], 0.9820117054643069), ('29-metal', [(29, 34)], 0.9973761863178063), ('34-\"],\\n', [(34, 38)], 0.9998792987698106), ('38- ', [(38, 41)], 1.0), ('41- \"', [(41, 43)], 1.0), ('43-type', [(43, 47)], 0.9999998063873693), ('47-\":', [(47, 49)], 1.0), ('49- \"', [(49, 51)], 0.9366606676356418), ('51-table', [(51, 56)], 0.46583453756679377), ('56- lamp', [(56, 61)], 0.9997951693480744), ('61-\",\\n', [(61, 64)], 0.9999772771533194), ('64- ', [(64, 67)], 1.0), ('67- \"', [(67, 69)], 1.0), ('69-color', [(69, 74)], 1.0), ('74-\":', [(74, 76)], 1.0), ('76- \"', [(76, 78)], 1.0), ('78-sil', [(78, 81)], 0.8931215422949286), ('81-ky', [(81, 83)], 0.9999723899264568), ('83- matte', [(83, 89)], 0.9999998063873693), ('89- grey', [(89, 94)], 0.999998137537802), ('94-\",\\n', [(94, 97)], 0.999999091165777), ('97- ', [(97, 100)], 1.0), ('100- \"', [(100, 102)], 1.0), ('102-price', [(102, 107)], 1.0), ('107-\":', [(107, 109)], 1.0), ('109- ', [(109, 110)], 0.9999976607241555), ('110-165', [(110, 113)], 0.0), ('113-.', [(113, 114)], 0.9999998063873693), ('114-00', [(114, 116)], 0.9388597312585776), ('116-,\\n', [(116, 118)], 0.9968273299853154), ('118- ', [(118, 121)], 1.0), ('121- \"', [(121, 123)], 1.0), ('123-summary', [(123, 130)], 1.0), ('130-\":', [(130, 132)], 1.0), ('132- \"', [(132, 134)], 0.9999996871837232), ('134-A', [(134, 135)], 0.12438188273141279), ('135- functional', [(135, 146)], 0.354933561939874), ('146- and', [(146, 150)], 0.9944640938740241), ('150- elegant', [(150, 158)], 0.6786980797185759), ('158- adjustable', [(158, 169)], 0.05042972424637176), ('169- table', [(169, 175)], 0.9911291136458064), ('175- lamp', [(175, 180)], 0.9999993295729128), ('180- designed', [(180, 189)], 0.5837374835494515), ('189- by', [(189, 192)], 0.9813930440515904), ('192- Maria', [(192, 198)], 0.9959209182415278), ('198- Ber', [(198, 202)], 1.0), ('202-nt', [(202, 204)], 1.0), ('204-sen', [(204, 207)], 1.0), ('207-.\"\\n', [(207, 210)], 0.9856653631776274), ('210-}\\n', [(210, 212)], 1.0), ('212-```', [(212, 215)], 0.9999723899264568)], preserve_density=False, with_probabilities=True, vec_name=None)], other=None), heatmap=None)], feature_importances=None, decision_tree=None, highlight_spaces=None, transition_features=None, image=None)" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "explanation = eli5.explain_prediction(client, prompt, model='gpt-4o', n=2)\n", + "explanation" + ] + }, + { + "cell_type": "markdown", + "id": "82b223cd-f23c-4737-a12a-873d71fa9b48", + "metadata": {}, + "source": [ + "We can obtain the original prediction from the explanation object via ``explanation.targets[0].target``,\n", + "e.g. use ``explanation.targets[0].target.message.content`` to get the prediction text." + ] + }, + { + "cell_type": "markdown", + "id": "a136a3c3-f840-403a-9a3a-9b7cd07065f4", + "metadata": {}, + "source": [ + "## Limitations\n", + "\n", + "Even though above the model confidence matched our expectations, it's not always the case.\n", + "For example, if we use \"Chain of Thought\" (https://arxiv.org/abs/2201.11903) reasoning,\n", + "asking the model first to think about the price estimate, it would be much more confident in the price in its final output,\n", + "but that does not reflect the real confidence of the model, as it's smeared over CoT:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "622fcc1f-fe36-47bf-a02c-ed5e62a25e6c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + "

To determine the most likely price for the Stay table lamp, we need to consider several factors:\n", + "\n", + "1. **Design and Brand**: The lamp is designed by Danish designer Maria Berntsen, which suggests a focus on high-quality design and potentially a higher price point due to the designer's reputation.\n", + "\n", + "2. **Functionality and Features**: The lamp has several advanced features, such as a flexible swivel mechanism and adjustable arm and head, which add to its functionality and likely increase its cost.\n", + "\n", + "3. **Materials and Finish**: The description mentions a "silky matte grey metal," indicating the use of quality materials that contribute to a modern and minimalistic aesthetic. This choice of materials can also influence the price.\n", + "\n", + "4. **Market Positioning**: Given the emphasis on both functionality and elegant appearance, the lamp is likely positioned in the mid to high-end market segment.\n", + "\n", + "Based on these factors, a reasonable estimate for the price of the Stay table lamp would be in the range of $150.00 to $300.00. For the purpose of this exercise, I will choose a mid-point estimate of $225.00.\n", + "\n", + "Now, I will provide the JSON with the extracted properties:\n", + "\n", + "```json\n", + "{\n", + " "materials": ["metal"],\n", + " "type": "table lamp",\n", + " "color": "silky matte grey",\n", + " "price": 225.00,\n", + " "summary": "A functional and elegant table lamp with adjustable features."\n", + "}\n", + "```

\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + "\n" + ], + "text/plain": [ + "Explanation(estimator='llm_logprobs', description=None, error=None, method=None, is_regression=False, targets=[TargetExplanation(target=Choice(finish_reason='stop', index=0, logprobs=ChoiceLogprobs(content=[ChatCompletionTokenLogprob(token='To', bytes=[84, 111], logprob=-0.4867655336856842, top_logprobs=[]), ChatCompletionTokenLogprob(token=' determine', bytes=[32, 100, 101, 116, 101, 114, 109, 105, 110, 101], logprob=-0.5725387930870056, top_logprobs=[]), ChatCompletionTokenLogprob(token=' the', bytes=[32, 116, 104, 101], logprob=-0.13688358664512634, top_logprobs=[]), ChatCompletionTokenLogprob(token=' most', bytes=[32, 109, 111, 115, 116], logprob=-0.19027188420295715, top_logprobs=[]), ChatCompletionTokenLogprob(token=' likely', bytes=[32, 108, 105, 107, 101, 108, 121], logprob=-0.0009278989746235311, top_logprobs=[]), ChatCompletionTokenLogprob(token=' price', bytes=[32, 112, 114, 105, 99, 101], logprob=-5.466968650580384e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token=' for', bytes=[32, 102, 111, 114], logprob=-0.4798658788204193, top_logprobs=[]), ChatCompletionTokenLogprob(token=' the', bytes=[32, 116, 104, 101], logprob=-0.009040074422955513, top_logprobs=[]), ChatCompletionTokenLogprob(token=' Stay', bytes=[32, 83, 116, 97, 121], logprob=-0.2057265341281891, top_logprobs=[]), ChatCompletionTokenLogprob(token=' table', bytes=[32, 116, 97, 98, 108, 101], logprob=-0.004193079192191362, top_logprobs=[]), ChatCompletionTokenLogprob(token=' lamp', bytes=[32, 108, 97, 109, 112], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=',', bytes=[44], logprob=-0.137336865067482, top_logprobs=[]), ChatCompletionTokenLogprob(token=' we', bytes=[32, 119, 101], logprob=-0.36271974444389343, top_logprobs=[]), ChatCompletionTokenLogprob(token=' need', bytes=[32, 110, 101, 101, 100], logprob=-0.7786848545074463, top_logprobs=[]), ChatCompletionTokenLogprob(token=' to', bytes=[32, 116, 111], logprob=-0.0005338519695214927, top_logprobs=[]), ChatCompletionTokenLogprob(token=' consider', bytes=[32, 99, 111, 110, 115, 105, 100, 101, 114], logprob=-0.04549144208431244, top_logprobs=[]), ChatCompletionTokenLogprob(token=' several', bytes=[32, 115, 101, 118, 101, 114, 97, 108], logprob=-0.6360411047935486, top_logprobs=[]), ChatCompletionTokenLogprob(token=' factors', bytes=[32, 102, 97, 99, 116, 111, 114, 115], logprob=-0.037374746054410934, top_logprobs=[]), ChatCompletionTokenLogprob(token=':\\n\\n', bytes=[58, 10, 10], logprob=-1.1546510457992554, top_logprobs=[]), ChatCompletionTokenLogprob(token='1', bytes=[49], logprob=-0.00035929924342781305, top_logprobs=[]), ChatCompletionTokenLogprob(token='.', bytes=[46], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' **', bytes=[32, 42, 42], logprob=-0.018017534166574478, top_logprobs=[]), ChatCompletionTokenLogprob(token='Design', bytes=[68, 101, 115, 105, 103, 110], logprob=-0.25278693437576294, top_logprobs=[]), ChatCompletionTokenLogprob(token=' and', bytes=[32, 97, 110, 100], logprob=-0.33452877402305603, top_logprobs=[]), ChatCompletionTokenLogprob(token=' Brand', bytes=[32, 66, 114, 97, 110, 100], logprob=-0.514303982257843, top_logprobs=[]), ChatCompletionTokenLogprob(token='**', bytes=[42, 42], logprob=-0.18277929723262787, top_logprobs=[]), ChatCompletionTokenLogprob(token=':', bytes=[58], logprob=-0.0037415295373648405, top_logprobs=[]), ChatCompletionTokenLogprob(token=' The', bytes=[32, 84, 104, 101], logprob=-0.18502825498580933, top_logprobs=[]), ChatCompletionTokenLogprob(token=' lamp', bytes=[32, 108, 97, 109, 112], logprob=-0.10684743523597717, top_logprobs=[]), ChatCompletionTokenLogprob(token=' is', bytes=[32, 105, 115], logprob=-0.001284875557757914, top_logprobs=[]), ChatCompletionTokenLogprob(token=' designed', bytes=[32, 100, 101, 115, 105, 103, 110, 101, 100], logprob=-0.003955867141485214, top_logprobs=[]), ChatCompletionTokenLogprob(token=' by', bytes=[32, 98, 121], logprob=-9.014684110297821e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token=' Danish', bytes=[32, 68, 97, 110, 105, 115, 104], logprob=-0.5383610129356384, top_logprobs=[]), ChatCompletionTokenLogprob(token=' designer', bytes=[32, 100, 101, 115, 105, 103, 110, 101, 114], logprob=-0.020705843344330788, top_logprobs=[]), ChatCompletionTokenLogprob(token=' Maria', bytes=[32, 77, 97, 114, 105, 97], logprob=-5.6100132496794686e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token=' Ber', bytes=[32, 66, 101, 114], logprob=-1.3856492842023727e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token='nt', bytes=[110, 116], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='sen', bytes=[115, 101, 110], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=',', bytes=[44], logprob=-0.24715355038642883, top_logprobs=[]), ChatCompletionTokenLogprob(token=' which', bytes=[32, 119, 104, 105, 99, 104], logprob=-0.5647792220115662, top_logprobs=[]), ChatCompletionTokenLogprob(token=' suggests', bytes=[32, 115, 117, 103, 103, 101, 115, 116, 115], logprob=-0.6731479167938232, top_logprobs=[]), ChatCompletionTokenLogprob(token=' a', bytes=[32, 97], logprob=-0.7282735705375671, top_logprobs=[]), ChatCompletionTokenLogprob(token=' focus', bytes=[32, 102, 111, 99, 117, 115], logprob=-0.7995526194572449, top_logprobs=[]), ChatCompletionTokenLogprob(token=' on', bytes=[32, 111, 110], logprob=-1.2664456789934775e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token=' high', bytes=[32, 104, 105, 103, 104], logprob=-0.5208733677864075, top_logprobs=[]), ChatCompletionTokenLogprob(token='-quality', bytes=[45, 113, 117, 97, 108, 105, 116, 121], logprob=-0.07403000444173813, top_logprobs=[]), ChatCompletionTokenLogprob(token=' design', bytes=[32, 100, 101, 115, 105, 103, 110], logprob=-0.32639598846435547, top_logprobs=[]), ChatCompletionTokenLogprob(token=' and', bytes=[32, 97, 110, 100], logprob=-0.2514616847038269, top_logprobs=[]), ChatCompletionTokenLogprob(token=' potentially', bytes=[32, 112, 111, 116, 101, 110, 116, 105, 97, 108, 108, 121], logprob=-1.488715410232544, top_logprobs=[]), ChatCompletionTokenLogprob(token=' a', bytes=[32, 97], logprob=-0.5769942998886108, top_logprobs=[]), ChatCompletionTokenLogprob(token=' higher', bytes=[32, 104, 105, 103, 104, 101, 114], logprob=-0.6764459609985352, top_logprobs=[]), ChatCompletionTokenLogprob(token=' price', bytes=[32, 112, 114, 105, 99, 101], logprob=-0.025465071201324463, top_logprobs=[]), ChatCompletionTokenLogprob(token=' point', bytes=[32, 112, 111, 105, 110, 116], logprob=-0.16209593415260315, top_logprobs=[]), ChatCompletionTokenLogprob(token=' due', bytes=[32, 100, 117, 101], logprob=-0.42360591888427734, top_logprobs=[]), ChatCompletionTokenLogprob(token=' to', bytes=[32, 116, 111], logprob=-2.4391956685576588e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token=' the', bytes=[32, 116, 104, 101], logprob=-0.27825748920440674, top_logprobs=[]), ChatCompletionTokenLogprob(token=' designer', bytes=[32, 100, 101, 115, 105, 103, 110, 101, 114], logprob=-0.29129624366760254, top_logprobs=[]), ChatCompletionTokenLogprob(token=\"'s\", bytes=[39, 115], logprob=-0.2024281769990921, top_logprobs=[]), ChatCompletionTokenLogprob(token=' reputation', bytes=[32, 114, 101, 112, 117, 116, 97, 116, 105, 111, 110], logprob=-0.2312728762626648, top_logprobs=[]), ChatCompletionTokenLogprob(token='.\\n\\n', bytes=[46, 10, 10], logprob=-0.3100598454475403, top_logprobs=[]), ChatCompletionTokenLogprob(token='2', bytes=[50], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='.', bytes=[46], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token=' **', bytes=[32, 42, 42], logprob=-4.320199877838604e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='Function', bytes=[70, 117, 110, 99, 116, 105, 111, 110], logprob=-1.0028660297393799, top_logprobs=[]), ChatCompletionTokenLogprob(token='ality', bytes=[97, 108, 105, 116, 121], logprob=-0.0005564896273426712, top_logprobs=[]), ChatCompletionTokenLogprob(token=' and', bytes=[32, 97, 110, 100], logprob=-0.6350793838500977, top_logprobs=[]), ChatCompletionTokenLogprob(token=' Features', bytes=[32, 70, 101, 97, 116, 117, 114, 101, 115], logprob=-0.011311288923025131, top_logprobs=[]), ChatCompletionTokenLogprob(token='**', bytes=[42, 42], logprob=-9.014684110297821e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token=':', bytes=[58], logprob=-0.0005530327325686812, top_logprobs=[]), ChatCompletionTokenLogprob(token=' The', bytes=[32, 84, 104, 101], logprob=-0.02379845269024372, top_logprobs=[]), ChatCompletionTokenLogprob(token=' lamp', bytes=[32, 108, 97, 109, 112], logprob=-0.1298796534538269, top_logprobs=[]), ChatCompletionTokenLogprob(token=' has', bytes=[32, 104, 97, 115], logprob=-1.265273094177246, top_logprobs=[]), ChatCompletionTokenLogprob(token=' several', bytes=[32, 115, 101, 118, 101, 114, 97, 108], logprob=-0.7500738501548767, top_logprobs=[]), ChatCompletionTokenLogprob(token=' advanced', bytes=[32, 97, 100, 118, 97, 110, 99, 101, 100], logprob=-0.8393288254737854, top_logprobs=[]), ChatCompletionTokenLogprob(token=' features', bytes=[32, 102, 101, 97, 116, 117, 114, 101, 115], logprob=-0.00549177173525095, top_logprobs=[]), ChatCompletionTokenLogprob(token=',', bytes=[44], logprob=-0.48477548360824585, top_logprobs=[]), ChatCompletionTokenLogprob(token=' such', bytes=[32, 115, 117, 99, 104], logprob=-0.296755850315094, top_logprobs=[]), ChatCompletionTokenLogprob(token=' as', bytes=[32, 97, 115], logprob=-6.704273118884885e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token=' a', bytes=[32, 97], logprob=-0.016932565718889236, top_logprobs=[]), ChatCompletionTokenLogprob(token=' flexible', bytes=[32, 102, 108, 101, 120, 105, 98, 108, 101], logprob=-0.002565906848758459, top_logprobs=[]), ChatCompletionTokenLogprob(token=' swivel', bytes=[32, 115, 119, 105, 118, 101, 108], logprob=-1.0445127372804563e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token=' mechanism', bytes=[32, 109, 101, 99, 104, 97, 110, 105, 115, 109], logprob=-1.3663626305060461e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token=' and', bytes=[32, 97, 110, 100], logprob=-0.6078916788101196, top_logprobs=[]), ChatCompletionTokenLogprob(token=' adjustable', bytes=[32, 97, 100, 106, 117, 115, 116, 97, 98, 108, 101], logprob=-0.3469345271587372, top_logprobs=[]), ChatCompletionTokenLogprob(token=' arm', bytes=[32, 97, 114, 109], logprob=-0.07993956655263901, top_logprobs=[]), ChatCompletionTokenLogprob(token=' and', bytes=[32, 97, 110, 100], logprob=-0.0018137017032131553, top_logprobs=[]), ChatCompletionTokenLogprob(token=' head', bytes=[32, 104, 101, 97, 100], logprob=-0.001930301426909864, top_logprobs=[]), ChatCompletionTokenLogprob(token=',', bytes=[44], logprob=-0.43073713779449463, top_logprobs=[]), ChatCompletionTokenLogprob(token=' which', bytes=[32, 119, 104, 105, 99, 104], logprob=-0.4856716990470886, top_logprobs=[]), ChatCompletionTokenLogprob(token=' add', bytes=[32, 97, 100, 100], logprob=-1.4381940364837646, top_logprobs=[]), ChatCompletionTokenLogprob(token=' to', bytes=[32, 116, 111], logprob=-0.1545192003250122, top_logprobs=[]), ChatCompletionTokenLogprob(token=' its', bytes=[32, 105, 116, 115], logprob=-0.07364879548549652, top_logprobs=[]), ChatCompletionTokenLogprob(token=' functionality', bytes=[32, 102, 117, 110, 99, 116, 105, 111, 110, 97, 108, 105, 116, 121], logprob=-0.27764394879341125, top_logprobs=[]), ChatCompletionTokenLogprob(token=' and', bytes=[32, 97, 110, 100], logprob=-0.3953467905521393, top_logprobs=[]), ChatCompletionTokenLogprob(token=' likely', bytes=[32, 108, 105, 107, 101, 108, 121], logprob=-1.4487597942352295, top_logprobs=[]), ChatCompletionTokenLogprob(token=' increase', bytes=[32, 105, 110, 99, 114, 101, 97, 115, 101], logprob=-0.11556302756071091, top_logprobs=[]), ChatCompletionTokenLogprob(token=' its', bytes=[32, 105, 116, 115], logprob=-0.43220198154449463, top_logprobs=[]), ChatCompletionTokenLogprob(token=' cost', bytes=[32, 99, 111, 115, 116], logprob=-1.0745759010314941, top_logprobs=[]), ChatCompletionTokenLogprob(token='.\\n\\n', bytes=[46, 10, 10], logprob=-0.1987321674823761, top_logprobs=[]), ChatCompletionTokenLogprob(token='3', bytes=[51], logprob=-5.512236498361744e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='.', bytes=[46], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token=' **', bytes=[32, 42, 42], logprob=-1.981667537620524e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token='Materials', bytes=[77, 97, 116, 101, 114, 105, 97, 108, 115], logprob=-0.3177346885204315, top_logprobs=[]), ChatCompletionTokenLogprob(token=' and', bytes=[32, 97, 110, 100], logprob=-0.7049893140792847, top_logprobs=[]), ChatCompletionTokenLogprob(token=' Finish', bytes=[32, 70, 105, 110, 105, 115, 104], logprob=-0.9479977488517761, top_logprobs=[]), ChatCompletionTokenLogprob(token='**', bytes=[42, 42], logprob=-6.2729995988775045e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token=':', bytes=[58], logprob=-1.4855663721391466e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token=' The', bytes=[32, 84, 104, 101], logprob=-0.179951474070549, top_logprobs=[]), ChatCompletionTokenLogprob(token=' description', bytes=[32, 100, 101, 115, 99, 114, 105, 112, 116, 105, 111, 110], logprob=-0.9577102661132812, top_logprobs=[]), ChatCompletionTokenLogprob(token=' mentions', bytes=[32, 109, 101, 110, 116, 105, 111, 110, 115], logprob=-0.1118556335568428, top_logprobs=[]), ChatCompletionTokenLogprob(token=' a', bytes=[32, 97], logprob=-0.3255186975002289, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=-0.1179909035563469, top_logprobs=[]), ChatCompletionTokenLogprob(token='sil', bytes=[115, 105, 108], logprob=-3.786196975852363e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token='ky', bytes=[107, 121], logprob=-4.0722858102526516e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token=' matte', bytes=[32, 109, 97, 116, 116, 101], logprob=-1.0280383548888494e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token=' grey', bytes=[32, 103, 114, 101, 121], logprob=-1.5451681974809617e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token=' metal', bytes=[32, 109, 101, 116, 97, 108], logprob=-0.0052392068319022655, top_logprobs=[]), ChatCompletionTokenLogprob(token=',\"', bytes=[44, 34], logprob=-0.4425547420978546, top_logprobs=[]), ChatCompletionTokenLogprob(token=' indicating', bytes=[32, 105, 110, 100, 105, 99, 97, 116, 105, 110, 103], logprob=-0.6408705711364746, top_logprobs=[]), ChatCompletionTokenLogprob(token=' the', bytes=[32, 116, 104, 101], logprob=-0.9325041770935059, top_logprobs=[]), ChatCompletionTokenLogprob(token=' use', bytes=[32, 117, 115, 101], logprob=-0.005283680744469166, top_logprobs=[]), ChatCompletionTokenLogprob(token=' of', bytes=[32, 111, 102], logprob=-4.246537173457909e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token=' quality', bytes=[32, 113, 117, 97, 108, 105, 116, 121], logprob=-0.41946297883987427, top_logprobs=[]), ChatCompletionTokenLogprob(token=' materials', bytes=[32, 109, 97, 116, 101, 114, 105, 97, 108, 115], logprob=-0.0028609856963157654, top_logprobs=[]), ChatCompletionTokenLogprob(token=' that', bytes=[32, 116, 104, 97, 116], logprob=-0.8825982213020325, top_logprobs=[]), ChatCompletionTokenLogprob(token=' contribute', bytes=[32, 99, 111, 110, 116, 114, 105, 98, 117, 116, 101], logprob=-1.0108187198638916, top_logprobs=[]), ChatCompletionTokenLogprob(token=' to', bytes=[32, 116, 111], logprob=-0.0030215606093406677, top_logprobs=[]), ChatCompletionTokenLogprob(token=' a', bytes=[32, 97], logprob=-0.11164658516645432, top_logprobs=[]), ChatCompletionTokenLogprob(token=' modern', bytes=[32, 109, 111, 100, 101, 114, 110], logprob=-0.07743553817272186, top_logprobs=[]), ChatCompletionTokenLogprob(token=' and', bytes=[32, 97, 110, 100], logprob=-0.07459418475627899, top_logprobs=[]), ChatCompletionTokenLogprob(token=' minimal', bytes=[32, 109, 105, 110, 105, 109, 97, 108], logprob=-0.16620802879333496, top_logprobs=[]), ChatCompletionTokenLogprob(token='istic', bytes=[105, 115, 116, 105, 99], logprob=-0.0004605783324223012, top_logprobs=[]), ChatCompletionTokenLogprob(token=' aesthetic', bytes=[32, 97, 101, 115, 116, 104, 101, 116, 105, 99], logprob=-1.272208333015442, top_logprobs=[]), ChatCompletionTokenLogprob(token='.', bytes=[46], logprob=-0.34661105275154114, top_logprobs=[]), ChatCompletionTokenLogprob(token=' This', bytes=[32, 84, 104, 105, 115], logprob=-0.5294309854507446, top_logprobs=[]), ChatCompletionTokenLogprob(token=' choice', bytes=[32, 99, 104, 111, 105, 99, 101], logprob=-0.6454590559005737, top_logprobs=[]), ChatCompletionTokenLogprob(token=' of', bytes=[32, 111, 102], logprob=-0.007842617109417915, top_logprobs=[]), ChatCompletionTokenLogprob(token=' materials', bytes=[32, 109, 97, 116, 101, 114, 105, 97, 108, 115], logprob=-0.5893488526344299, top_logprobs=[]), ChatCompletionTokenLogprob(token=' can', bytes=[32, 99, 97, 110], logprob=-1.3226159811019897, top_logprobs=[]), ChatCompletionTokenLogprob(token=' also', bytes=[32, 97, 108, 115, 111], logprob=-0.7278075814247131, top_logprobs=[]), ChatCompletionTokenLogprob(token=' influence', bytes=[32, 105, 110, 102, 108, 117, 101, 110, 99, 101], logprob=-1.14788019657135, top_logprobs=[]), ChatCompletionTokenLogprob(token=' the', bytes=[32, 116, 104, 101], logprob=-0.05610622465610504, top_logprobs=[]), ChatCompletionTokenLogprob(token=' price', bytes=[32, 112, 114, 105, 99, 101], logprob=-0.1288062334060669, top_logprobs=[]), ChatCompletionTokenLogprob(token='.\\n\\n', bytes=[46, 10, 10], logprob=-0.6564987301826477, top_logprobs=[]), ChatCompletionTokenLogprob(token='4', bytes=[52], logprob=-0.0038703777827322483, top_logprobs=[]), ChatCompletionTokenLogprob(token='.', bytes=[46], logprob=-3.128163257315464e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token=' **', bytes=[32, 42, 42], logprob=-1.1637164789135568e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token='Market', bytes=[77, 97, 114, 107, 101, 116], logprob=-0.13202142715454102, top_logprobs=[]), ChatCompletionTokenLogprob(token=' Position', bytes=[32, 80, 111, 115, 105, 116, 105, 111, 110], logprob=-0.6024223566055298, top_logprobs=[]), ChatCompletionTokenLogprob(token='ing', bytes=[105, 110, 103], logprob=-0.6018260717391968, top_logprobs=[]), ChatCompletionTokenLogprob(token='**', bytes=[42, 42], logprob=-0.01463598944246769, top_logprobs=[]), ChatCompletionTokenLogprob(token=':', bytes=[58], logprob=-1.4259644558478612e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token=' Given', bytes=[32, 71, 105, 118, 101, 110], logprob=-1.009255051612854, top_logprobs=[]), ChatCompletionTokenLogprob(token=' the', bytes=[32, 116, 104, 101], logprob=-0.42789575457572937, top_logprobs=[]), ChatCompletionTokenLogprob(token=' emphasis', bytes=[32, 101, 109, 112, 104, 97, 115, 105, 115], logprob=-0.9409775733947754, top_logprobs=[]), ChatCompletionTokenLogprob(token=' on', bytes=[32, 111, 110], logprob=-1.3306015716807451e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token=' both', bytes=[32, 98, 111, 116, 104], logprob=-0.6611020565032959, top_logprobs=[]), ChatCompletionTokenLogprob(token=' functionality', bytes=[32, 102, 117, 110, 99, 116, 105, 111, 110, 97, 108, 105, 116, 121], logprob=-0.45676717162132263, top_logprobs=[]), ChatCompletionTokenLogprob(token=' and', bytes=[32, 97, 110, 100], logprob=-0.00039659533649683, top_logprobs=[]), ChatCompletionTokenLogprob(token=' elegant', bytes=[32, 101, 108, 101, 103, 97, 110, 116], logprob=-1.149277925491333, top_logprobs=[]), ChatCompletionTokenLogprob(token=' appearance', bytes=[32, 97, 112, 112, 101, 97, 114, 97, 110, 99, 101], logprob=-0.4769769608974457, top_logprobs=[]), ChatCompletionTokenLogprob(token=',', bytes=[44], logprob=-0.005541327875107527, top_logprobs=[]), ChatCompletionTokenLogprob(token=' the', bytes=[32, 116, 104, 101], logprob=-0.6089863777160645, top_logprobs=[]), ChatCompletionTokenLogprob(token=' lamp', bytes=[32, 108, 97, 109, 112], logprob=-0.389377236366272, top_logprobs=[]), ChatCompletionTokenLogprob(token=' is', bytes=[32, 105, 115], logprob=-0.43932950496673584, top_logprobs=[]), ChatCompletionTokenLogprob(token=' likely', bytes=[32, 108, 105, 107, 101, 108, 121], logprob=-0.06283710896968842, top_logprobs=[]), ChatCompletionTokenLogprob(token=' positioned', bytes=[32, 112, 111, 115, 105, 116, 105, 111, 110, 101, 100], logprob=-0.04982662573456764, top_logprobs=[]), ChatCompletionTokenLogprob(token=' in', bytes=[32, 105, 110], logprob=-0.6074603199958801, top_logprobs=[]), ChatCompletionTokenLogprob(token=' the', bytes=[32, 116, 104, 101], logprob=-0.10160448402166367, top_logprobs=[]), ChatCompletionTokenLogprob(token=' mid', bytes=[32, 109, 105, 100], logprob=-0.30626994371414185, top_logprobs=[]), ChatCompletionTokenLogprob(token=' to', bytes=[32, 116, 111], logprob=-0.47068873047828674, top_logprobs=[]), ChatCompletionTokenLogprob(token=' high', bytes=[32, 104, 105, 103, 104], logprob=-0.1460588276386261, top_logprobs=[]), ChatCompletionTokenLogprob(token='-end', bytes=[45, 101, 110, 100], logprob=-0.020427661016583443, top_logprobs=[]), ChatCompletionTokenLogprob(token=' market', bytes=[32, 109, 97, 114, 107, 101, 116], logprob=-0.2641751170158386, top_logprobs=[]), ChatCompletionTokenLogprob(token=' segment', bytes=[32, 115, 101, 103, 109, 101, 110, 116], logprob=-0.4092211127281189, top_logprobs=[]), ChatCompletionTokenLogprob(token='.\\n\\n', bytes=[46, 10, 10], logprob=-0.32233819365501404, top_logprobs=[]), ChatCompletionTokenLogprob(token='Based', bytes=[66, 97, 115, 101, 100], logprob=-0.7752827405929565, top_logprobs=[]), ChatCompletionTokenLogprob(token=' on', bytes=[32, 111, 110], logprob=-1.9504606825648807e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token=' these', bytes=[32, 116, 104, 101, 115, 101], logprob=-0.08208250999450684, top_logprobs=[]), ChatCompletionTokenLogprob(token=' factors', bytes=[32, 102, 97, 99, 116, 111, 114, 115], logprob=-0.7664922475814819, top_logprobs=[]), ChatCompletionTokenLogprob(token=',', bytes=[44], logprob=-0.22567568719387054, top_logprobs=[]), ChatCompletionTokenLogprob(token=' a', bytes=[32, 97], logprob=-0.9796017408370972, top_logprobs=[]), ChatCompletionTokenLogprob(token=' reasonable', bytes=[32, 114, 101, 97, 115, 111, 110, 97, 98, 108, 101], logprob=-0.12573164701461792, top_logprobs=[]), ChatCompletionTokenLogprob(token=' estimate', bytes=[32, 101, 115, 116, 105, 109, 97, 116, 101], logprob=-0.4213482737541199, top_logprobs=[]), ChatCompletionTokenLogprob(token=' for', bytes=[32, 102, 111, 114], logprob=-0.011164778843522072, top_logprobs=[]), ChatCompletionTokenLogprob(token=' the', bytes=[32, 116, 104, 101], logprob=-0.08224985748529434, top_logprobs=[]), ChatCompletionTokenLogprob(token=' price', bytes=[32, 112, 114, 105, 99, 101], logprob=-0.10537097603082657, top_logprobs=[]), ChatCompletionTokenLogprob(token=' of', bytes=[32, 111, 102], logprob=-0.17389218509197235, top_logprobs=[]), ChatCompletionTokenLogprob(token=' the', bytes=[32, 116, 104, 101], logprob=-0.3174235224723816, top_logprobs=[]), ChatCompletionTokenLogprob(token=' Stay', bytes=[32, 83, 116, 97, 121], logprob=-0.021571654826402664, top_logprobs=[]), ChatCompletionTokenLogprob(token=' table', bytes=[32, 116, 97, 98, 108, 101], logprob=-0.0016024599317461252, top_logprobs=[]), ChatCompletionTokenLogprob(token=' lamp', bytes=[32, 108, 97, 109, 112], logprob=-9.088346359931165e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token=' would', bytes=[32, 119, 111, 117, 108, 100], logprob=-0.6761465072631836, top_logprobs=[]), ChatCompletionTokenLogprob(token=' be', bytes=[32, 98, 101], logprob=-0.5130773782730103, top_logprobs=[]), ChatCompletionTokenLogprob(token=' in', bytes=[32, 105, 110], logprob=-0.6392275094985962, top_logprobs=[]), ChatCompletionTokenLogprob(token=' the', bytes=[32, 116, 104, 101], logprob=-0.016515925526618958, top_logprobs=[]), ChatCompletionTokenLogprob(token=' range', bytes=[32, 114, 97, 110, 103, 101], logprob=-0.020542005077004433, top_logprobs=[]), ChatCompletionTokenLogprob(token=' of', bytes=[32, 111, 102], logprob=-0.01836426928639412, top_logprobs=[]), ChatCompletionTokenLogprob(token=' $', bytes=[32, 36], logprob=-0.09238871186971664, top_logprobs=[]), ChatCompletionTokenLogprob(token='150', bytes=[49, 53, 48], logprob=-0.10822010785341263, top_logprobs=[]), ChatCompletionTokenLogprob(token='.', bytes=[46], logprob=-0.5784227848052979, top_logprobs=[]), ChatCompletionTokenLogprob(token='00', bytes=[48, 48], logprob=-9.610702363715973e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token=' to', bytes=[32, 116, 111], logprob=-0.001433324534446001, top_logprobs=[]), ChatCompletionTokenLogprob(token=' $', bytes=[32, 36], logprob=-1.8624639324116288e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token='300', bytes=[51, 48, 48], logprob=-0.19296149909496307, top_logprobs=[]), ChatCompletionTokenLogprob(token='.', bytes=[46], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='00', bytes=[48, 48], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='.', bytes=[46], logprob=-0.13142132759094238, top_logprobs=[]), ChatCompletionTokenLogprob(token=' For', bytes=[32, 70, 111, 114], logprob=-0.570335865020752, top_logprobs=[]), ChatCompletionTokenLogprob(token=' the', bytes=[32, 116, 104, 101], logprob=-0.2939893901348114, top_logprobs=[]), ChatCompletionTokenLogprob(token=' purpose', bytes=[32, 112, 117, 114, 112, 111, 115, 101], logprob=-0.37512460350990295, top_logprobs=[]), ChatCompletionTokenLogprob(token=' of', bytes=[32, 111, 102], logprob=-3.8889261304575484e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token=' this', bytes=[32, 116, 104, 105, 115], logprob=-0.2848525643348694, top_logprobs=[]), ChatCompletionTokenLogprob(token=' exercise', bytes=[32, 101, 120, 101, 114, 99, 105, 115, 101], logprob=-0.8632019758224487, top_logprobs=[]), ChatCompletionTokenLogprob(token=',', bytes=[44], logprob=-0.023284848779439926, top_logprobs=[]), ChatCompletionTokenLogprob(token=' I', bytes=[32, 73], logprob=-1.3395649194717407, top_logprobs=[]), ChatCompletionTokenLogprob(token=' will', bytes=[32, 119, 105, 108, 108], logprob=-0.005582586862146854, top_logprobs=[]), ChatCompletionTokenLogprob(token=' choose', bytes=[32, 99, 104, 111, 111, 115, 101], logprob=-1.483843207359314, top_logprobs=[]), ChatCompletionTokenLogprob(token=' a', bytes=[32, 97], logprob=-0.20085424184799194, top_logprobs=[]), ChatCompletionTokenLogprob(token=' mid', bytes=[32, 109, 105, 100], logprob=-1.272668719291687, top_logprobs=[]), ChatCompletionTokenLogprob(token='-point', bytes=[45, 112, 111, 105, 110, 116], logprob=-0.6398953199386597, top_logprobs=[]), ChatCompletionTokenLogprob(token=' estimate', bytes=[32, 101, 115, 116, 105, 109, 97, 116, 101], logprob=-0.31783151626586914, top_logprobs=[]), ChatCompletionTokenLogprob(token=' of', bytes=[32, 111, 102], logprob=-0.5164365768432617, top_logprobs=[]), ChatCompletionTokenLogprob(token=' $', bytes=[32, 36], logprob=-0.008019437082111835, top_logprobs=[]), ChatCompletionTokenLogprob(token='225', bytes=[50, 50, 53], logprob=-0.05688877031207085, top_logprobs=[]), ChatCompletionTokenLogprob(token='.', bytes=[46], logprob=-4.723352049040841e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token='00', bytes=[48, 48], logprob=-0.002398482523858547, top_logprobs=[]), ChatCompletionTokenLogprob(token='.\\n\\n', bytes=[46, 10, 10], logprob=-0.4111771583557129, top_logprobs=[]), ChatCompletionTokenLogprob(token='Now', bytes=[78, 111, 119], logprob=-0.44121289253234863, top_logprobs=[]), ChatCompletionTokenLogprob(token=',', bytes=[44], logprob=-0.010635088197886944, top_logprobs=[]), ChatCompletionTokenLogprob(token=' I', bytes=[32, 73], logprob=-1.21943998336792, top_logprobs=[]), ChatCompletionTokenLogprob(token=' will', bytes=[32, 119, 105, 108, 108], logprob=-0.0025467700324952602, top_logprobs=[]), ChatCompletionTokenLogprob(token=' provide', bytes=[32, 112, 114, 111, 118, 105, 100, 101], logprob=-0.2521243095397949, top_logprobs=[]), ChatCompletionTokenLogprob(token=' the', bytes=[32, 116, 104, 101], logprob=-0.018794996663928032, top_logprobs=[]), ChatCompletionTokenLogprob(token=' JSON', bytes=[32, 74, 83, 79, 78], logprob=-0.33000504970550537, top_logprobs=[]), ChatCompletionTokenLogprob(token=' with', bytes=[32, 119, 105, 116, 104], logprob=-0.8988541960716248, top_logprobs=[]), ChatCompletionTokenLogprob(token=' the', bytes=[32, 116, 104, 101], logprob=-0.16055524349212646, top_logprobs=[]), ChatCompletionTokenLogprob(token=' extracted', bytes=[32, 101, 120, 116, 114, 97, 99, 116, 101, 100], logprob=-0.0017182581359520555, top_logprobs=[]), ChatCompletionTokenLogprob(token=' properties', bytes=[32, 112, 114, 111, 112, 101, 114, 116, 105, 101, 115], logprob=-0.5211189985275269, top_logprobs=[]), ChatCompletionTokenLogprob(token=':\\n\\n', bytes=[58, 10, 10], logprob=-0.23949961364269257, top_logprobs=[]), ChatCompletionTokenLogprob(token='```', bytes=[96, 96, 96], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='json', bytes=[106, 115, 111, 110], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='\\n', bytes=[10], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='{\\n', bytes=[123, 10], logprob=-1.5048530030981055e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32], logprob=-0.5760262608528137, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='materials', bytes=[109, 97, 116, 101, 114, 105, 97, 108, 115], logprob=-3.128163257315464e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='\":', bytes=[34, 58], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' [\"', bytes=[32, 91, 34], logprob=-0.000488811288960278, top_logprobs=[]), ChatCompletionTokenLogprob(token='metal', bytes=[109, 101, 116, 97, 108], logprob=-0.014525678008794785, top_logprobs=[]), ChatCompletionTokenLogprob(token='\"],\\n', bytes=[34, 93, 44, 10], logprob=-0.0014908155426383018, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='type', bytes=[116, 121, 112, 101], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='\":', bytes=[34, 58], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=-0.15258395671844482, top_logprobs=[]), ChatCompletionTokenLogprob(token='table', bytes=[116, 97, 98, 108, 101], logprob=-0.4851100742816925, top_logprobs=[]), ChatCompletionTokenLogprob(token=' lamp', bytes=[32, 108, 97, 109, 112], logprob=-0.005258422344923019, top_logprobs=[]), ChatCompletionTokenLogprob(token='\",\\n', bytes=[34, 44, 10], logprob=-0.00018816311785485595, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='color', bytes=[99, 111, 108, 111, 114], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='\":', bytes=[34, 58], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='sil', bytes=[115, 105, 108], logprob=-0.2694111466407776, top_logprobs=[]), ChatCompletionTokenLogprob(token='ky', bytes=[107, 121], logprob=-0.000431861262768507, top_logprobs=[]), ChatCompletionTokenLogprob(token=' matte', bytes=[32, 109, 97, 116, 116, 101], logprob=-1.0280383548888494e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token=' grey', bytes=[32, 103, 114, 101, 121], logprob=-2.935296834039036e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token='\",\\n', bytes=[34, 44, 10], logprob=-4.4849443838757e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='price', bytes=[112, 114, 105, 99, 101], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='\":', bytes=[34, 58], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='225', bytes=[50, 50, 53], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='.', bytes=[46], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='00', bytes=[48, 48], logprob=-5.512236498361744e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token=',\\n', bytes=[44, 10], logprob=-6.2729995988775045e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='summary', bytes=[115, 117, 109, 109, 97, 114, 121], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='\":', bytes=[34, 58], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='A', bytes=[65], logprob=-1.227637529373169, top_logprobs=[]), ChatCompletionTokenLogprob(token=' functional', bytes=[32, 102, 117, 110, 99, 116, 105, 111, 110, 97, 108], logprob=-0.41765180230140686, top_logprobs=[]), ChatCompletionTokenLogprob(token=' and', bytes=[32, 97, 110, 100], logprob=-0.0053070466965436935, top_logprobs=[]), ChatCompletionTokenLogprob(token=' elegant', bytes=[32, 101, 108, 101, 103, 97, 110, 116], logprob=-0.419668585062027, top_logprobs=[]), ChatCompletionTokenLogprob(token=' table', bytes=[32, 116, 97, 98, 108, 101], logprob=-0.03026655875146389, top_logprobs=[]), ChatCompletionTokenLogprob(token=' lamp', bytes=[32, 108, 97, 109, 112], logprob=-1.2664456789934775e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token=' with', bytes=[32, 119, 105, 116, 104], logprob=-0.8116578459739685, top_logprobs=[]), ChatCompletionTokenLogprob(token=' adjustable', bytes=[32, 97, 100, 106, 117, 115, 116, 97, 98, 108, 101], logprob=-0.3303641378879547, top_logprobs=[]), ChatCompletionTokenLogprob(token=' features', bytes=[32, 102, 101, 97, 116, 117, 114, 101, 115], logprob=-0.2633365988731384, top_logprobs=[]), ChatCompletionTokenLogprob(token='.\"\\n', bytes=[46, 34, 10], logprob=-0.8170788288116455, top_logprobs=[]), ChatCompletionTokenLogprob(token='}\\n', bytes=[125, 10], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='```', bytes=[96, 96, 96], logprob=-0.00861456897109747, top_logprobs=[])], refusal=None), message=ChatCompletionMessage(content='To determine the most likely price for the Stay table lamp, we need to consider several factors:\\n\\n1. **Design and Brand**: The lamp is designed by Danish designer Maria Berntsen, which suggests a focus on high-quality design and potentially a higher price point due to the designer\\'s reputation.\\n\\n2. **Functionality and Features**: The lamp has several advanced features, such as a flexible swivel mechanism and adjustable arm and head, which add to its functionality and likely increase its cost.\\n\\n3. **Materials and Finish**: The description mentions a \"silky matte grey metal,\" indicating the use of quality materials that contribute to a modern and minimalistic aesthetic. This choice of materials can also influence the price.\\n\\n4. **Market Positioning**: Given the emphasis on both functionality and elegant appearance, the lamp is likely positioned in the mid to high-end market segment.\\n\\nBased on these factors, a reasonable estimate for the price of the Stay table lamp would be in the range of $150.00 to $300.00. For the purpose of this exercise, I will choose a mid-point estimate of $225.00.\\n\\nNow, I will provide the JSON with the extracted properties:\\n\\n```json\\n{\\n \"materials\": [\"metal\"],\\n \"type\": \"table lamp\",\\n \"color\": \"silky matte grey\",\\n \"price\": 225.00,\\n \"summary\": \"A functional and elegant table lamp with adjustable features.\"\\n}\\n```', refusal=None, role='assistant', annotations=[], audio=None, function_call=None, tool_calls=None)), feature_weights=None, proba=None, score=None, weighted_spans=WeightedSpans(docs_weighted_spans=[DocWeightedSpans(document='To determine the most likely price for the Stay table lamp, we need to consider several factors:\\n\\n1. **Design and Brand**: The lamp is designed by Danish designer Maria Berntsen, which suggests a focus on high-quality design and potentially a higher price point due to the designer\\'s reputation.\\n\\n2. **Functionality and Features**: The lamp has several advanced features, such as a flexible swivel mechanism and adjustable arm and head, which add to its functionality and likely increase its cost.\\n\\n3. **Materials and Finish**: The description mentions a \"silky matte grey metal,\" indicating the use of quality materials that contribute to a modern and minimalistic aesthetic. This choice of materials can also influence the price.\\n\\n4. **Market Positioning**: Given the emphasis on both functionality and elegant appearance, the lamp is likely positioned in the mid to high-end market segment.\\n\\nBased on these factors, a reasonable estimate for the price of the Stay table lamp would be in the range of $150.00 to $300.00. For the purpose of this exercise, I will choose a mid-point estimate of $225.00.\\n\\nNow, I will provide the JSON with the extracted properties:\\n\\n```json\\n{\\n \"materials\": [\"metal\"],\\n \"type\": \"table lamp\",\\n \"color\": \"silky matte grey\",\\n \"price\": 225.00,\\n \"summary\": \"A functional and elegant table lamp with adjustable features.\"\\n}\\n```', spans=[('0-To', [(0, 2)], 0.6146111216564284), ('2- determine', [(2, 12)], 0.5640915076225902), ('12- the', [(12, 16)], 0.8720717410202131), ('16- most', [(16, 21)], 0.8267343273804134), ('21- likely', [(21, 28)], 0.9990725313905079), ('28- price', [(28, 34)], 0.9999453318078543), ('34- for', [(34, 38)], 0.6188663893302934), ('38- the', [(38, 42)], 0.9910006641973534), ('42- Stay', [(42, 47)], 0.8140556622880858), ('47- table', [(47, 53)], 0.995815699490175), ('53- lamp', [(53, 58)], 1.0), ('58-,', [(58, 59)], 0.871676539292237), ('59- we', [(59, 62)], 0.6957814027811717), ('62- need', [(62, 67)], 0.45900927850987233), ('67- to', [(67, 70)], 0.9994662905040868), ('70- consider', [(70, 79)], 0.9555277798631413), ('79- several', [(79, 87)], 0.5293840570390912), ('87- factors', [(87, 95)], 0.9633150691778885), ('95-:\\n\\n', [(95, 98)], 0.31516749673874944), ('98-1', [(98, 99)], 0.9996407652968153), ('99-.', [(99, 100)], 1.0), ('100- **', [(100, 103)], 0.9821438111341367), ('103-Design', [(103, 109)], 0.7766333380635712), ('109- and', [(109, 113)], 0.7156752516754369), ('113- Brand', [(113, 119)], 0.5979166103934268), ('119-**', [(119, 121)], 0.832951970257082), ('121-:', [(121, 122)], 0.9962654612627951), ('122- The', [(122, 126)], 0.8310808013441299), ('126- lamp', [(126, 131)], 0.8986627668051506), ('131- is', [(131, 134)], 0.9987159495414205), ('134- designed', [(134, 143)], 0.996051946993646), ('143- by', [(143, 146)], 0.9999909853565219), ('146- Danish', [(146, 153)], 0.5837041523612825), ('153- designer', [(153, 162)], 0.9795070507143964), ('162- Maria', [(162, 168)], 0.9999439014410862), ('168- Ber', [(168, 172)], 0.9999986143516758), ('172-nt', [(172, 174)], 1.0), ('174-sen', [(174, 177)], 1.0), ('177-,', [(177, 178)], 0.7810207582843168), ('178- which', [(178, 184)], 0.5684856420202337), ('184- suggests', [(184, 193)], 0.5101002944601217), ('193- a', [(193, 195)], 0.4827416905613928), ('195- focus', [(195, 201)], 0.44953003012622045), ('201- on', [(201, 204)], 0.9999987335551229), ('204- high', [(204, 209)], 0.5940015395508474), ('209--quality', [(209, 217)], 0.9286438299835773), ('217- design', [(217, 224)], 0.7215194174885157), ('224- and', [(224, 228)], 0.7776632534368243), ('228- potentially', [(228, 240)], 0.22566235297795414), ('240- a', [(240, 242)], 0.5615837848101857), ('242- higher', [(242, 249)], 0.50842073229768), ('249- price', [(249, 255)], 0.9748564289350192), ('255- point', [(255, 261)], 0.8503596220972705), ('261- due', [(261, 265)], 0.6546818288250628), ('265- to', [(265, 268)], 0.9999756083407958), ('268- the', [(268, 272)], 0.7571018508607605), ('272- designer', [(272, 281)], 0.7472942640299097), (\"281-'s\", [(281, 283)], 0.8167451415785988), ('283- reputation', [(283, 294)], 0.7935229029250314), ('294-.\\n\\n', [(294, 297)], 0.7334030640763327), ('297-2', [(297, 298)], 0.9999998063873693), ('298-.', [(298, 299)], 0.9999998063873693), ('299- **', [(299, 302)], 0.9999995679801056), ('302-Function', [(302, 310)], 0.3668265972142673), ('310-ality', [(310, 315)], 0.9994436651842916), ('315- and', [(315, 319)], 0.5298934216679309), ('319- Features', [(319, 328)], 0.9887524431817728), ('328-**', [(328, 330)], 0.9999909853565219), ('330-:', [(330, 331)], 0.9994471201618464), ('331- The', [(331, 335)], 0.976482497346603), ('335- lamp', [(335, 340)], 0.878201113031994), ('340- has', [(340, 344)], 0.2821622287559633), ('344- several', [(344, 352)], 0.4723316696860116), ('352- advanced', [(352, 361)], 0.432000373794314), ('361- features', [(361, 370)], 0.9945232804761032), ('370-,', [(370, 371)], 0.6158354463963893), ('371- such', [(371, 376)], 0.7432254484713126), ('376- as', [(376, 379)], 0.9999993295729128), ('379- a', [(379, 381)], 0.983209984457927), ('381- flexible', [(381, 390)], 0.9974373822764221), ('390- swivel', [(390, 397)], 0.9999895549271773), ('397- mechanism', [(397, 407)], 0.9999863364670418), ('407- and', [(407, 411)], 0.54449763567669), ('411- adjustable', [(411, 422)], 0.7068516063257719), ('422- arm', [(422, 426)], 0.9231721351755007), ('426- and', [(426, 430)], 0.9981879420598055), ('430- head', [(430, 435)], 0.9980715604067301), ('435-,', [(435, 436)], 0.6500297565747712), ('436- which', [(436, 442)], 0.6152837724078406), ('442- add', [(442, 446)], 0.23735602817342305), ('446- to', [(446, 449)], 0.8568270406265007), ('449- its', [(449, 453)], 0.9289979048126857), ('453- functionality', [(453, 467)], 0.7575665059692703), ('467- and', [(467, 471)], 0.6734464538810316), ('471- likely', [(471, 478)], 0.23486138398942794), ('478- increase', [(478, 487)], 0.8908644214561648), ('487- its', [(487, 491)], 0.6490782616144022), ('491- cost', [(491, 496)], 0.34144253002420566), ('496-.\\n\\n', [(496, 499)], 0.8197694248417015), ('499-3', [(499, 500)], 0.999999448776502), ('500-.', [(500, 501)], 0.9999998063873693), ('501- **', [(501, 504)], 0.9999980183344259), ('504-Materials', [(504, 513)], 0.7277958553972388), ('513- and', [(513, 517)], 0.4941138542894019), ('517- Finish', [(517, 524)], 0.3875161518546848), ('524-**', [(524, 526)], 0.9999937270200764), ('526-:', [(526, 527)], 0.9999851444466235), ('527- The', [(527, 531)], 0.8353107446580723), ('531- description', [(531, 543)], 0.38377061328861434), ('543- mentions', [(543, 552)], 0.8941733368069738), ('552- a', [(552, 554)], 0.7221526776900313), ('554- \"', [(554, 556)], 0.8887041366249525), ('556-sil', [(556, 559)], 0.9999621387469968), ('559-ky', [(559, 561)], 0.9999592779710618), ('561- matte', [(561, 567)], 0.9999989719621736), ('567- grey', [(567, 572)], 0.9999845484374018), ('572- metal', [(572, 578)], 0.9947744938748232), ('578-,\"', [(578, 580)], 0.6423931740555772), ('580- indicating', [(580, 591)], 0.5268335782361324), ('591- the', [(591, 595)], 0.3935669140799408), ('595- use', [(595, 599)], 0.9947302533447409), ('599- of', [(599, 602)], 0.9999957534718431), ('602- quality', [(602, 610)], 0.6573997626208862), ('610- materials', [(610, 620)], 0.9971431030230762), ('620- that', [(620, 625)], 0.4137066127216595), ('625- contribute', [(625, 636)], 0.3639209082932298), ('636- to', [(636, 639)], 0.9969829997106666), ('639- a', [(639, 641)], 0.8943602818433601), ('641- modern', [(641, 648)], 0.9254866810364094), ('648- and', [(648, 652)], 0.9281200551810903), ('652- minimal', [(652, 660)], 0.8468700425105089), ('660-istic', [(660, 665)], 0.9995395277174958), ('665- aesthetic', [(665, 675)], 0.2802121363030958), ('675-.', [(675, 676)], 0.707080291715045), ('676- This', [(676, 681)], 0.5889399897764819), ('681- choice', [(681, 688)], 0.524421747944048), ('688- of', [(688, 691)], 0.9921880559740168), ('691- materials', [(691, 701)], 0.5546883510265505), ('701- can', [(701, 705)], 0.26643739431884367), ('705- also', [(705, 710)], 0.4829666953543355), ('710- influence', [(710, 720)], 0.31730868900557396), ('720- the', [(720, 724)], 0.9454387016552429), ('724- price', [(724, 730)], 0.879144297838864), ('730-.\\n\\n', [(730, 733)], 0.518664142181721), ('733-4', [(733, 734)], 0.996137102475771), ('734-.', [(734, 735)], 0.9999996871837232), ('735- **', [(735, 738)], 0.9999883629029224), ('738-Market', [(738, 744)], 0.8763222177866138), ('744- Position', [(744, 753)], 0.5474838274644224), ('753-ing', [(753, 756)], 0.5478103811351317), ('756-**', [(756, 758)], 0.9854705960228307), ('758-:', [(758, 759)], 0.9999857404571098), ('759- Given', [(759, 765)], 0.36449040499933816), ('765- the', [(765, 769)], 0.6518793666979878), ('769- emphasis', [(769, 778)], 0.3902461545703039), ('778- on', [(778, 781)], 0.9999866940728078), ('781- both', [(781, 786)], 0.5162820488673656), ('786- functionality', [(786, 800)], 0.6333277795781326), ('800- and', [(800, 804)], 0.9996034832970381), ('804- elegant', [(804, 812)], 0.31686548728447544), ('812- appearance', [(812, 823)], 0.6206568285050391), ('823-,', [(823, 824)], 0.9944739969624857), ('824- the', [(824, 828)], 0.5439019008452516), ('828- lamp', [(828, 833)], 0.6774786522178488), ('833- is', [(833, 836)], 0.6444683891047422), ('836- likely', [(836, 843)], 0.9390964316163658), ('843- positioned', [(843, 854)], 0.9513943575006002), ('854- in', [(854, 857)], 0.54473256019591), ('857- the', [(857, 861)], 0.9033867849265456), ('861- mid', [(861, 865)], 0.7361878633446882), ('865- to', [(865, 868)], 0.6245719583723212), ('868- high', [(868, 873)], 0.8641068683228259), ('873--end', [(873, 877)], 0.9797795701690998), ('877- market', [(877, 884)], 0.7678390661968376), ('884- segment', [(884, 892)], 0.6641673602282229), ('892-.\\n\\n', [(892, 895)], 0.7244531434391206), ('895-Based', [(895, 900)], 0.4605735397518602), ('900- on', [(900, 903)], 0.999980495583388), ('903- these', [(903, 909)], 0.9211959476879744), ('909- factors', [(909, 917)], 0.4646400553870793), ('917-,', [(917, 918)], 0.797976853817776), ('918- a', [(918, 920)], 0.3754605997035223), ('920- reasonable', [(920, 931)], 0.8818514625063554), ('931- estimate', [(931, 940)], 0.6561615377680553), ('940- for', [(940, 944)], 0.9888973159932366), ('944- the', [(944, 948)], 0.921041800756016), ('948- price', [(948, 954)], 0.8999905857135383), ('954- of', [(954, 957)], 0.8403874990751723), ('957- the', [(957, 961)], 0.7280223559951703), ('961- Stay', [(961, 966)], 0.9786593492908856), ('966- table', [(966, 972)], 0.9983988233216248), ('972- lamp', [(972, 977)], 0.999999091165777), ('977- would', [(977, 983)], 0.5085730035830615), ('983- be', [(983, 986)], 0.5986504672742095), ('986- in', [(986, 989)], 0.5276999098012592), ('989- the', [(989, 993)], 0.9836197146039899), ('993- range', [(993, 999)], 0.9796675445995804), ('999- of', [(999, 1002)], 0.9818033264145193), ('1002- $', [(1002, 1004)], 0.9117506723462433), ('1004-150', [(1004, 1007)], 0.8974300432893171), ('1007-.', [(1007, 1008)], 0.5607821435466721), ('1008-00', [(1008, 1010)], 0.9999903893438189), ('1010- to', [(1010, 1013)], 0.9985677021845654), ('1013- $', [(1013, 1015)], 0.999998137537802), ('1015-300', [(1015, 1018)], 0.8245137180523799), ('1018-.', [(1018, 1019)], 1.0), ('1019-00', [(1019, 1021)], 0.9999998063873693), ('1021-.', [(1021, 1022)], 0.8768482561890008), ('1022- For', [(1022, 1026)], 0.5653355303798986), ('1026- the', [(1026, 1030)], 0.7452843987683389), ('1030- purpose', [(1030, 1038)], 0.6872036454697331), ('1038- of', [(1038, 1041)], 0.9999961110814314), ('1041- this', [(1041, 1046)], 0.752125136214928), ('1046- exercise', [(1046, 1055)], 0.4218092945139154), ('1055-,', [(1055, 1056)], 0.9769841513907835), ('1056- I', [(1056, 1058)], 0.2619596173187755), ('1058- will', [(1058, 1063)], 0.994432966819171), ('1063- choose', [(1063, 1070)], 0.22676450852494492), ('1070- a', [(1070, 1072)], 0.8180316576472635), ('1072- mid', [(1072, 1076)], 0.2800831601728028), ('1076--point', [(1076, 1082)], 0.5273476239354553), ('1082- estimate', [(1082, 1091)], 0.7277253879770776), ('1091- of', [(1091, 1094)], 0.5966428553518711), ('1094- $', [(1094, 1096)], 0.992012632818668), ('1096-225', [(1096, 1099)], 0.9446991421135749), ('1099-.', [(1099, 1100)], 0.9999952766591059), ('1100-00', [(1100, 1102)], 0.9976043915370959), ('1102-.\\n\\n', [(1102, 1105)], 0.6628694883294014), ('1105-Now', [(1105, 1108)], 0.6432557476496426), ('1108-,', [(1108, 1109)], 0.9894212644040474), ('1109- I', [(1109, 1111)], 0.2953955470312666), ('1111- will', [(1111, 1116)], 0.9974564702349815), ('1116- provide', [(1116, 1124)], 0.7771481251383277), ('1124- the', [(1124, 1128)], 0.9813805279044621), ('1128- JSON', [(1128, 1133)], 0.7189201030879576), ('1133- with', [(1133, 1138)], 0.40703577584172224), ('1138- the', [(1138, 1142)], 0.8516707730047014), ('1142- extracted', [(1142, 1152)], 0.9982832172244211), ('1152- properties', [(1152, 1163)], 0.5938556524303757), ('1163-:\\n\\n', [(1163, 1166)], 0.7870215774134383), ('1166-```', [(1166, 1169)], 1.0), ('1169-json', [(1169, 1173)], 1.0), ('1173-\\n', [(1173, 1174)], 1.0), ('1174-{\\n', [(1174, 1176)], 0.9999984951481292), ('1176- ', [(1176, 1177)], 0.5621276830506429), ('1177- \"', [(1177, 1179)], 1.0), ('1179-materials', [(1179, 1188)], 0.9999996871837232), ('1188-\":', [(1188, 1190)], 1.0), ('1190- [\"', [(1190, 1193)], 0.9995113081598144), ('1193-metal', [(1193, 1198)], 0.9855793106932458), ('1198-\"],\\n', [(1198, 1202)], 0.9985102951708279), ('1202- ', [(1202, 1203)], 1.0), ('1203- \"', [(1203, 1205)], 1.0), ('1205-type', [(1205, 1209)], 1.0), ('1209-\":', [(1209, 1211)], 1.0), ('1211- \"', [(1211, 1213)], 0.8584868151948569), ('1213-table', [(1213, 1218)], 0.6156294280674749), ('1218- lamp', [(1218, 1223)], 0.9947553789562354), ('1223-\",\\n', [(1223, 1226)], 0.9998118545837144), ('1226- ', [(1226, 1227)], 1.0), ('1227- \"', [(1227, 1229)], 1.0), ('1229-color', [(1229, 1234)], 0.9999998063873693), ('1234-\":', [(1234, 1236)], 1.0), ('1236- \"', [(1236, 1238)], 0.9999998063873693), ('1238-sil', [(1238, 1241)], 0.7638291452926999), ('1241-ky', [(1241, 1243)], 0.9995682319758841), ('1243- matte', [(1243, 1249)], 0.9999989719621736), ('1249- grey', [(1249, 1254)], 0.999997064707474), ('1254-\",\\n', [(1254, 1257)], 0.9999955150656735), ('1257- ', [(1257, 1258)], 0.9999998063873693), ('1258- \"', [(1258, 1260)], 1.0), ('1260-price', [(1260, 1265)], 1.0), ('1265-\":', [(1265, 1267)], 1.0), ('1267- ', [(1267, 1268)], 0.9999998063873693), ('1268-225', [(1268, 1271)], 1.0), ('1271-.', [(1271, 1272)], 1.0), ('1272-00', [(1272, 1274)], 0.999999448776502), ('1274-,\\n', [(1274, 1276)], 0.9999937270200764), ('1276- ', [(1276, 1277)], 0.9999998063873693), ('1277- \"', [(1277, 1279)], 1.0), ('1279-summary', [(1279, 1286)], 1.0), ('1286-\":', [(1286, 1288)], 1.0), ('1288- \"', [(1288, 1290)], 0.9999998063873693), ('1290-A', [(1290, 1291)], 0.2929839266343186), ('1291- functional', [(1291, 1302)], 0.6585915085526952), ('1302- and', [(1302, 1306)], 0.9947070107968573), ('1306- elegant', [(1306, 1314)], 0.6572646110337294), ('1314- table', [(1314, 1320)], 0.9701868872724675), ('1320- lamp', [(1320, 1325)], 0.9999987335551229), ('1325- with', [(1325, 1330)], 0.4441211710669766), ('1330- adjustable', [(1330, 1341)], 0.7186619937195989), ('1341- features', [(1341, 1350)], 0.7684831831986731), ('1350-.\"\\n', [(1350, 1353)], 0.44172011175144665), ('1353-}\\n', [(1353, 1355)], 0.9999998063873693), ('1355-```', [(1355, 1358)], 0.991422430108248)], preserve_density=False, with_probabilities=True, vec_name=None)], other=None), heatmap=None)], feature_importances=None, decision_tree=None, highlight_spaces=None, transition_features=None, image=None)" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "prompt_cot = prompt + \"\"\"\n", + "Before outputting the JSON with extracted results, provide analysis of the most likely price.\n", + "\"\"\"\n", + "eli5.explain_prediction(client, prompt_cot, model='gpt-4o', temperature=0)" + ] + }, + { + "cell_type": "markdown", + "id": "6f0adca2-4729-41b8-8d76-a91cf7d7082f", + "metadata": {}, + "source": [ + "We can see that the model has already committed to a specific price point as part of it's analysis,\n", + "and it's condfidence is very high in a particular prediction, but this is not indicative of the true confidence.\n", + "\n", + "Finally, an interesting point is that if we leave the temperature at its default value of 1, the analysis would show up a lot of less condfient predictions, which is expected given the sampling performed at non-zero temperatures:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "3caaeca2-1d58-406a-ba22-3fb15942ffe8", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + "

Analysis of Price:\n", + "\n", + "The product description does not provide an explicit price for the Stay table lamp. To estimate the price, it's important to consider factors such as the design origin (Danish design by Maria Berntsen), functionality (flexible swivel mechanism and adjustable arm and head), material quality (silky matte grey metal), and intended use (task lighting in modern, minimalistic settings).\n", + "\n", + "Based on these characteristics, the Stay table lamp is likely positioned as a mid-to-high-end product in the market. Danish-designed lighting products known for combining aesthetics with functionality often range from approximately $150 to $400. Given these aspects, a reasonable estimate for the price could be around $250.00.\n", + "\n", + "Now, I will provide the JSON dict with all extracted information:\n", + "\n", + "```json\n", + "{\n", + " "materials": ["metal"],\n", + " "type": "table lamp",\n", + " "color": "silky matte grey",\n", + " "price": 250.00,\n", + " "summary": "A flexible and elegant table lamp designed by Maria Berntsen."\n", + "}\n", + "```

\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + "\n" + ], + "text/plain": [ + "Explanation(estimator='llm_logprobs', description=None, error=None, method=None, is_regression=False, targets=[TargetExplanation(target=Choice(finish_reason='stop', index=0, logprobs=ChoiceLogprobs(content=[ChatCompletionTokenLogprob(token='Analysis', bytes=[65, 110, 97, 108, 121, 115, 105, 115], logprob=-3.470536947250366, top_logprobs=[]), ChatCompletionTokenLogprob(token=' of', bytes=[32, 111, 102], logprob=-0.07548464089632034, top_logprobs=[]), ChatCompletionTokenLogprob(token=' Price', bytes=[32, 80, 114, 105, 99, 101], logprob=-2.8842244148254395, top_logprobs=[]), ChatCompletionTokenLogprob(token=':\\n\\n', bytes=[58, 10, 10], logprob=-0.4149337112903595, top_logprobs=[]), ChatCompletionTokenLogprob(token='The', bytes=[84, 104, 101], logprob=-0.1381458044052124, top_logprobs=[]), ChatCompletionTokenLogprob(token=' product', bytes=[32, 112, 114, 111, 100, 117, 99, 116], logprob=-0.0961286872625351, top_logprobs=[]), ChatCompletionTokenLogprob(token=' description', bytes=[32, 100, 101, 115, 99, 114, 105, 112, 116, 105, 111, 110], logprob=-0.05404112488031387, top_logprobs=[]), ChatCompletionTokenLogprob(token=' does', bytes=[32, 100, 111, 101, 115], logprob=-0.7267906665802002, top_logprobs=[]), ChatCompletionTokenLogprob(token=' not', bytes=[32, 110, 111, 116], logprob=-8.41866585687967e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token=' provide', bytes=[32, 112, 114, 111, 118, 105, 100, 101], logprob=-0.8855213522911072, top_logprobs=[]), ChatCompletionTokenLogprob(token=' an', bytes=[32, 97, 110], logprob=-1.4068679809570312, top_logprobs=[]), ChatCompletionTokenLogprob(token=' explicit', bytes=[32, 101, 120, 112, 108, 105, 99, 105, 116], logprob=-0.016328897327184677, top_logprobs=[]), ChatCompletionTokenLogprob(token=' price', bytes=[32, 112, 114, 105, 99, 101], logprob=-0.0034691598266363144, top_logprobs=[]), ChatCompletionTokenLogprob(token=' for', bytes=[32, 102, 111, 114], logprob=-0.14249683916568756, top_logprobs=[]), ChatCompletionTokenLogprob(token=' the', bytes=[32, 116, 104, 101], logprob=-0.00014883324911352247, top_logprobs=[]), ChatCompletionTokenLogprob(token=' Stay', bytes=[32, 83, 116, 97, 121], logprob=-0.20818568766117096, top_logprobs=[]), ChatCompletionTokenLogprob(token=' table', bytes=[32, 116, 97, 98, 108, 101], logprob=-0.0077700018882751465, top_logprobs=[]), ChatCompletionTokenLogprob(token=' lamp', bytes=[32, 108, 97, 109, 112], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='.', bytes=[46], logprob=-0.046941064298152924, top_logprobs=[]), ChatCompletionTokenLogprob(token=' To', bytes=[32, 84, 111], logprob=-1.4835844039916992, top_logprobs=[]), ChatCompletionTokenLogprob(token=' estimate', bytes=[32, 101, 115, 116, 105, 109, 97, 116, 101], logprob=-0.0776246190071106, top_logprobs=[]), ChatCompletionTokenLogprob(token=' the', bytes=[32, 116, 104, 101], logprob=-0.6596805453300476, top_logprobs=[]), ChatCompletionTokenLogprob(token=' price', bytes=[32, 112, 114, 105, 99, 101], logprob=-0.09459269791841507, top_logprobs=[]), ChatCompletionTokenLogprob(token=',', bytes=[44], logprob=-0.016335463151335716, top_logprobs=[]), ChatCompletionTokenLogprob(token=\" it's\", bytes=[32, 105, 116, 39, 115], logprob=-4.817591190338135, top_logprobs=[]), ChatCompletionTokenLogprob(token=' important', bytes=[32, 105, 109, 112, 111, 114, 116, 97, 110, 116], logprob=-0.6164944171905518, top_logprobs=[]), ChatCompletionTokenLogprob(token=' to', bytes=[32, 116, 111], logprob=-3.523948907968588e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token=' consider', bytes=[32, 99, 111, 110, 115, 105, 100, 101, 114], logprob=-0.00870734453201294, top_logprobs=[]), ChatCompletionTokenLogprob(token=' factors', bytes=[32, 102, 97, 99, 116, 111, 114, 115], logprob=-1.8121271133422852, top_logprobs=[]), ChatCompletionTokenLogprob(token=' such', bytes=[32, 115, 117, 99, 104], logprob=-0.13879328966140747, top_logprobs=[]), ChatCompletionTokenLogprob(token=' as', bytes=[32, 97, 115], logprob=-9.088346359931165e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token=' the', bytes=[32, 116, 104, 101], logprob=-0.42100611329078674, top_logprobs=[]), ChatCompletionTokenLogprob(token=' design', bytes=[32, 100, 101, 115, 105, 103, 110], logprob=-0.9122723340988159, top_logprobs=[]), ChatCompletionTokenLogprob(token=' origin', bytes=[32, 111, 114, 105, 103, 105, 110], logprob=-0.8151019811630249, top_logprobs=[]), ChatCompletionTokenLogprob(token=' (', bytes=[32, 40], logprob=-0.9830595254898071, top_logprobs=[]), ChatCompletionTokenLogprob(token='D', bytes=[68], logprob=-0.11297745257616043, top_logprobs=[]), ChatCompletionTokenLogprob(token='anish', bytes=[97, 110, 105, 115, 104], logprob=-4.320199877838604e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token=' design', bytes=[32, 100, 101, 115, 105, 103, 110], logprob=-0.5272119641304016, top_logprobs=[]), ChatCompletionTokenLogprob(token=' by', bytes=[32, 98, 121], logprob=-0.2019457370042801, top_logprobs=[]), ChatCompletionTokenLogprob(token=' Maria', bytes=[32, 77, 97, 114, 105, 97], logprob=-0.001441899803467095, top_logprobs=[]), ChatCompletionTokenLogprob(token=' Ber', bytes=[32, 66, 101, 114], logprob=-1.5048530030981055e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token='nt', bytes=[110, 116], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='sen', bytes=[115, 101, 110], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='),', bytes=[41, 44], logprob=-0.005479914136230946, top_logprobs=[]), ChatCompletionTokenLogprob(token=' functionality', bytes=[32, 102, 117, 110, 99, 116, 105, 111, 110, 97, 108, 105, 116, 121], logprob=-2.8578741550445557, top_logprobs=[]), ChatCompletionTokenLogprob(token=' (', bytes=[32, 40], logprob=-0.3838485777378082, top_logprobs=[]), ChatCompletionTokenLogprob(token='flex', bytes=[102, 108, 101, 120], logprob=-0.6801329255104065, top_logprobs=[]), ChatCompletionTokenLogprob(token='ible', bytes=[105, 98, 108, 101], logprob=-0.01604997180402279, top_logprobs=[]), ChatCompletionTokenLogprob(token=' swivel', bytes=[32, 115, 119, 105, 118, 101, 108], logprob=-0.015337996184825897, top_logprobs=[]), ChatCompletionTokenLogprob(token=' mechanism', bytes=[32, 109, 101, 99, 104, 97, 110, 105, 115, 109], logprob=-0.0020417715422809124, top_logprobs=[]), ChatCompletionTokenLogprob(token=' and', bytes=[32, 97, 110, 100], logprob=-1.8754100799560547, top_logprobs=[]), ChatCompletionTokenLogprob(token=' adjustable', bytes=[32, 97, 100, 106, 117, 115, 116, 97, 98, 108, 101], logprob=-0.05741465464234352, top_logprobs=[]), ChatCompletionTokenLogprob(token=' arm', bytes=[32, 97, 114, 109], logprob=-0.07884681969881058, top_logprobs=[]), ChatCompletionTokenLogprob(token=' and', bytes=[32, 97, 110, 100], logprob=-0.06441891938447952, top_logprobs=[]), ChatCompletionTokenLogprob(token=' head', bytes=[32, 104, 101, 97, 100], logprob=-0.003179259831085801, top_logprobs=[]), ChatCompletionTokenLogprob(token='),', bytes=[41, 44], logprob=-0.03575617074966431, top_logprobs=[]), ChatCompletionTokenLogprob(token=' material', bytes=[32, 109, 97, 116, 101, 114, 105, 97, 108], logprob=-1.3489422798156738, top_logprobs=[]), ChatCompletionTokenLogprob(token=' quality', bytes=[32, 113, 117, 97, 108, 105, 116, 121], logprob=-0.98789381980896, top_logprobs=[]), ChatCompletionTokenLogprob(token=' (', bytes=[32, 40], logprob=-0.0027631523553282022, top_logprobs=[]), ChatCompletionTokenLogprob(token='sil', bytes=[115, 105, 108], logprob=-0.046541180461645126, top_logprobs=[]), ChatCompletionTokenLogprob(token='ky', bytes=[107, 121], logprob=-0.0009175319573841989, top_logprobs=[]), ChatCompletionTokenLogprob(token=' matte', bytes=[32, 109, 97, 116, 116, 101], logprob=-1.4259644558478612e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token=' grey', bytes=[32, 103, 114, 101, 121], logprob=-0.015260986983776093, top_logprobs=[]), ChatCompletionTokenLogprob(token=' metal', bytes=[32, 109, 101, 116, 97, 108], logprob=-0.00024382755509577692, top_logprobs=[]), ChatCompletionTokenLogprob(token='),', bytes=[41, 44], logprob=-0.031130028888583183, top_logprobs=[]), ChatCompletionTokenLogprob(token=' and', bytes=[32, 97, 110, 100], logprob=-0.01216877344995737, top_logprobs=[]), ChatCompletionTokenLogprob(token=' intended', bytes=[32, 105, 110, 116, 101, 110, 100, 101, 100], logprob=-1.7925525903701782, top_logprobs=[]), ChatCompletionTokenLogprob(token=' use', bytes=[32, 117, 115, 101], logprob=-0.11203160881996155, top_logprobs=[]), ChatCompletionTokenLogprob(token=' (', bytes=[32, 40], logprob=-0.020648395642638206, top_logprobs=[]), ChatCompletionTokenLogprob(token='task', bytes=[116, 97, 115, 107], logprob=-0.4322299659252167, top_logprobs=[]), ChatCompletionTokenLogprob(token=' lighting', bytes=[32, 108, 105, 103, 104, 116, 105, 110, 103], logprob=-1.4855663721391466e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token=' in', bytes=[32, 105, 110], logprob=-0.6959487199783325, top_logprobs=[]), ChatCompletionTokenLogprob(token=' modern', bytes=[32, 109, 111, 100, 101, 114, 110], logprob=-2.7436435222625732, top_logprobs=[]), ChatCompletionTokenLogprob(token=',', bytes=[44], logprob=-1.7304409742355347, top_logprobs=[]), ChatCompletionTokenLogprob(token=' minimal', bytes=[32, 109, 105, 110, 105, 109, 97, 108], logprob=-0.18132318556308746, top_logprobs=[]), ChatCompletionTokenLogprob(token='istic', bytes=[105, 115, 116, 105, 99], logprob=-0.0023766038939356804, top_logprobs=[]), ChatCompletionTokenLogprob(token=' settings', bytes=[32, 115, 101, 116, 116, 105, 110, 103, 115], logprob=-0.2906411588191986, top_logprobs=[]), ChatCompletionTokenLogprob(token=').\\n\\n', bytes=[41, 46, 10, 10], logprob=-1.7185794115066528, top_logprobs=[]), ChatCompletionTokenLogprob(token='Based', bytes=[66, 97, 115, 101, 100], logprob=-2.036790609359741, top_logprobs=[]), ChatCompletionTokenLogprob(token=' on', bytes=[32, 111, 110], logprob=-2.2723104848410003e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token=' these', bytes=[32, 116, 104, 101, 115, 101], logprob=-0.6593130826950073, top_logprobs=[]), ChatCompletionTokenLogprob(token=' characteristics', bytes=[32, 99, 104, 97, 114, 97, 99, 116, 101, 114, 105, 115, 116, 105, 99, 115], logprob=-2.027945041656494, top_logprobs=[]), ChatCompletionTokenLogprob(token=',', bytes=[44], logprob=-0.31632760167121887, top_logprobs=[]), ChatCompletionTokenLogprob(token=' the', bytes=[32, 116, 104, 101], logprob=-0.8887858390808105, top_logprobs=[]), ChatCompletionTokenLogprob(token=' Stay', bytes=[32, 83, 116, 97, 121], logprob=-0.46331724524497986, top_logprobs=[]), ChatCompletionTokenLogprob(token=' table', bytes=[32, 116, 97, 98, 108, 101], logprob=-0.048772674053907394, top_logprobs=[]), ChatCompletionTokenLogprob(token=' lamp', bytes=[32, 108, 97, 109, 112], logprob=-2.2200749754119897e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token=' is', bytes=[32, 105, 115], logprob=-1.146453619003296, top_logprobs=[]), ChatCompletionTokenLogprob(token=' likely', bytes=[32, 108, 105, 107, 101, 108, 121], logprob=-0.19884903728961945, top_logprobs=[]), ChatCompletionTokenLogprob(token=' positioned', bytes=[32, 112, 111, 115, 105, 116, 105, 111, 110, 101, 100], logprob=-0.16601024568080902, top_logprobs=[]), ChatCompletionTokenLogprob(token=' as', bytes=[32, 97, 115], logprob=-0.6005222797393799, top_logprobs=[]), ChatCompletionTokenLogprob(token=' a', bytes=[32, 97], logprob=-0.0025139504577964544, top_logprobs=[]), ChatCompletionTokenLogprob(token=' mid', bytes=[32, 109, 105, 100], logprob=-1.7088682651519775, top_logprobs=[]), ChatCompletionTokenLogprob(token='-to', bytes=[45, 116, 111], logprob=-1.050847053527832, top_logprobs=[]), ChatCompletionTokenLogprob(token='-high', bytes=[45, 104, 105, 103, 104], logprob=-0.02905040979385376, top_logprobs=[]), ChatCompletionTokenLogprob(token='-end', bytes=[45, 101, 110, 100], logprob=-0.18556329607963562, top_logprobs=[]), ChatCompletionTokenLogprob(token=' product', bytes=[32, 112, 114, 111, 100, 117, 99, 116], logprob=-0.22854799032211304, top_logprobs=[]), ChatCompletionTokenLogprob(token=' in', bytes=[32, 105, 110], logprob=-0.9800060987472534, top_logprobs=[]), ChatCompletionTokenLogprob(token=' the', bytes=[32, 116, 104, 101], logprob=-0.039520345628261566, top_logprobs=[]), ChatCompletionTokenLogprob(token=' market', bytes=[32, 109, 97, 114, 107, 101, 116], logprob=-0.8923746943473816, top_logprobs=[]), ChatCompletionTokenLogprob(token='.', bytes=[46], logprob=-0.1461196094751358, top_logprobs=[]), ChatCompletionTokenLogprob(token=' Danish', bytes=[32, 68, 97, 110, 105, 115, 104], logprob=-1.4255707263946533, top_logprobs=[]), ChatCompletionTokenLogprob(token='-designed', bytes=[45, 100, 101, 115, 105, 103, 110, 101, 100], logprob=-1.3932698965072632, top_logprobs=[]), ChatCompletionTokenLogprob(token=' lighting', bytes=[32, 108, 105, 103, 104, 116, 105, 110, 103], logprob=-0.3129201829433441, top_logprobs=[]), ChatCompletionTokenLogprob(token=' products', bytes=[32, 112, 114, 111, 100, 117, 99, 116, 115], logprob=-0.8966972827911377, top_logprobs=[]), ChatCompletionTokenLogprob(token=' known', bytes=[32, 107, 110, 111, 119, 110], logprob=-4.667842864990234, top_logprobs=[]), ChatCompletionTokenLogprob(token=' for', bytes=[32, 102, 111, 114], logprob=-1.1637164789135568e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token=' combining', bytes=[32, 99, 111, 109, 98, 105, 110, 105, 110, 103], logprob=-3.5495126247406006, top_logprobs=[]), ChatCompletionTokenLogprob(token=' aesthetics', bytes=[32, 97, 101, 115, 116, 104, 101, 116, 105, 99, 115], logprob=-1.151599407196045, top_logprobs=[]), ChatCompletionTokenLogprob(token=' with', bytes=[32, 119, 105, 116, 104], logprob=-0.7611759305000305, top_logprobs=[]), ChatCompletionTokenLogprob(token=' functionality', bytes=[32, 102, 117, 110, 99, 116, 105, 111, 110, 97, 108, 105, 116, 121], logprob=-0.03216484934091568, top_logprobs=[]), ChatCompletionTokenLogprob(token=' often', bytes=[32, 111, 102, 116, 101, 110], logprob=-1.234926700592041, top_logprobs=[]), ChatCompletionTokenLogprob(token=' range', bytes=[32, 114, 97, 110, 103, 101], logprob=-1.017974615097046, top_logprobs=[]), ChatCompletionTokenLogprob(token=' from', bytes=[32, 102, 114, 111, 109], logprob=-0.47531887888908386, top_logprobs=[]), ChatCompletionTokenLogprob(token=' approximately', bytes=[32, 97, 112, 112, 114, 111, 120, 105, 109, 97, 116, 101, 108, 121], logprob=-1.0552136898040771, top_logprobs=[]), ChatCompletionTokenLogprob(token=' $', bytes=[32, 36], logprob=-0.02605084888637066, top_logprobs=[]), ChatCompletionTokenLogprob(token='150', bytes=[49, 53, 48], logprob=-0.3372423052787781, top_logprobs=[]), ChatCompletionTokenLogprob(token=' to', bytes=[32, 116, 111], logprob=-0.0553513802587986, top_logprobs=[]), ChatCompletionTokenLogprob(token=' $', bytes=[32, 36], logprob=-0.004928105045109987, top_logprobs=[]), ChatCompletionTokenLogprob(token='400', bytes=[52, 48, 48], logprob=-0.8789671659469604, top_logprobs=[]), ChatCompletionTokenLogprob(token='.', bytes=[46], logprob=-0.9540044069290161, top_logprobs=[]), ChatCompletionTokenLogprob(token=' Given', bytes=[32, 71, 105, 118, 101, 110], logprob=-0.39220085740089417, top_logprobs=[]), ChatCompletionTokenLogprob(token=' these', bytes=[32, 116, 104, 101, 115, 101], logprob=-3.0127131938934326, top_logprobs=[]), ChatCompletionTokenLogprob(token=' aspects', bytes=[32, 97, 115, 112, 101, 99, 116, 115], logprob=-3.4703633785247803, top_logprobs=[]), ChatCompletionTokenLogprob(token=',', bytes=[44], logprob=-0.22813519835472107, top_logprobs=[]), ChatCompletionTokenLogprob(token=' a', bytes=[32, 97], logprob=-0.5484052896499634, top_logprobs=[]), ChatCompletionTokenLogprob(token=' reasonable', bytes=[32, 114, 101, 97, 115, 111, 110, 97, 98, 108, 101], logprob=-0.062040284276008606, top_logprobs=[]), ChatCompletionTokenLogprob(token=' estimate', bytes=[32, 101, 115, 116, 105, 109, 97, 116, 101], logprob=-0.6718961596488953, top_logprobs=[]), ChatCompletionTokenLogprob(token=' for', bytes=[32, 102, 111, 114], logprob=-0.032594285905361176, top_logprobs=[]), ChatCompletionTokenLogprob(token=' the', bytes=[32, 116, 104, 101], logprob=-0.062239013612270355, top_logprobs=[]), ChatCompletionTokenLogprob(token=' price', bytes=[32, 112, 114, 105, 99, 101], logprob=-0.5471123456954956, top_logprobs=[]), ChatCompletionTokenLogprob(token=' could', bytes=[32, 99, 111, 117, 108, 100], logprob=-2.299600124359131, top_logprobs=[]), ChatCompletionTokenLogprob(token=' be', bytes=[32, 98, 101], logprob=-0.0175681971013546, top_logprobs=[]), ChatCompletionTokenLogprob(token=' around', bytes=[32, 97, 114, 111, 117, 110, 100], logprob=-0.2351464182138443, top_logprobs=[]), ChatCompletionTokenLogprob(token=' $', bytes=[32, 36], logprob=-0.022104566916823387, top_logprobs=[]), ChatCompletionTokenLogprob(token='250', bytes=[50, 53, 48], logprob=-0.08193402737379074, top_logprobs=[]), ChatCompletionTokenLogprob(token='.', bytes=[46], logprob=-0.021702788770198822, top_logprobs=[]), ChatCompletionTokenLogprob(token='00', bytes=[48, 48], logprob=-0.0022797922138124704, top_logprobs=[]), ChatCompletionTokenLogprob(token='.\\n\\n', bytes=[46, 10, 10], logprob=-0.25367406010627747, top_logprobs=[]), ChatCompletionTokenLogprob(token='Now', bytes=[78, 111, 119], logprob=-1.1820061206817627, top_logprobs=[]), ChatCompletionTokenLogprob(token=',', bytes=[44], logprob=-0.021479368209838867, top_logprobs=[]), ChatCompletionTokenLogprob(token=' I', bytes=[32, 73], logprob=-1.3105272054672241, top_logprobs=[]), ChatCompletionTokenLogprob(token=' will', bytes=[32, 119, 105, 108, 108], logprob=-0.004302287939935923, top_logprobs=[]), ChatCompletionTokenLogprob(token=' provide', bytes=[32, 112, 114, 111, 118, 105, 100, 101], logprob=-0.27623647451400757, top_logprobs=[]), ChatCompletionTokenLogprob(token=' the', bytes=[32, 116, 104, 101], logprob=-0.02480592206120491, top_logprobs=[]), ChatCompletionTokenLogprob(token=' JSON', bytes=[32, 74, 83, 79, 78], logprob=-0.22695498168468475, top_logprobs=[]), ChatCompletionTokenLogprob(token=' dict', bytes=[32, 100, 105, 99, 116], logprob=-1.3176720142364502, top_logprobs=[]), ChatCompletionTokenLogprob(token=' with', bytes=[32, 119, 105, 116, 104], logprob=-0.03229481726884842, top_logprobs=[]), ChatCompletionTokenLogprob(token=' all', bytes=[32, 97, 108, 108], logprob=-7.828518867492676, top_logprobs=[]), ChatCompletionTokenLogprob(token=' extracted', bytes=[32, 101, 120, 116, 114, 97, 99, 116, 101, 100], logprob=-0.7467106580734253, top_logprobs=[]), ChatCompletionTokenLogprob(token=' information', bytes=[32, 105, 110, 102, 111, 114, 109, 97, 116, 105, 111, 110], logprob=-1.9947469234466553, top_logprobs=[]), ChatCompletionTokenLogprob(token=':\\n\\n', bytes=[58, 10, 10], logprob=-0.6308677196502686, top_logprobs=[]), ChatCompletionTokenLogprob(token='```', bytes=[96, 96, 96], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='json', bytes=[106, 115, 111, 110], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='\\n', bytes=[10], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='{\\n', bytes=[123, 10], logprob=-6.2729995988775045e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32, 32, 32], logprob=-0.474151074886322, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='materials', bytes=[109, 97, 116, 101, 114, 105, 97, 108, 115], logprob=-4.320199877838604e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='\":', bytes=[34, 58], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' [\"', bytes=[32, 91, 34], logprob=-0.0010337610729038715, top_logprobs=[]), ChatCompletionTokenLogprob(token='metal', bytes=[109, 101, 116, 97, 108], logprob=-0.028111685067415237, top_logprobs=[]), ChatCompletionTokenLogprob(token='\"],\\n', bytes=[34, 93, 44, 10], logprob=-0.000873347744345665, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32, 32, 32], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='type', bytes=[116, 121, 112, 101], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='\":', bytes=[34, 58], logprob=-3.128163257315464e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=-0.09915617108345032, top_logprobs=[]), ChatCompletionTokenLogprob(token='table', bytes=[116, 97, 98, 108, 101], logprob=-0.38992148637771606, top_logprobs=[]), ChatCompletionTokenLogprob(token=' lamp', bytes=[32, 108, 97, 109, 112], logprob=-0.005958187393844128, top_logprobs=[]), ChatCompletionTokenLogprob(token='\",\\n', bytes=[34, 44, 10], logprob=-0.0003460712905507535, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32, 32, 32], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='color', bytes=[99, 111, 108, 111, 114], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='\":', bytes=[34, 58], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='sil', bytes=[115, 105, 108], logprob=-0.2278972715139389, top_logprobs=[]), ChatCompletionTokenLogprob(token='ky', bytes=[107, 121], logprob=-0.00029744720086455345, top_logprobs=[]), ChatCompletionTokenLogprob(token=' matte', bytes=[32, 109, 97, 116, 116, 101], logprob=-9.088346359931165e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token=' grey', bytes=[32, 103, 114, 101, 121], logprob=-2.696889623621246e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token='\",\\n', bytes=[34, 44, 10], logprob=-9.253090865968261e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32, 32, 32], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='price', bytes=[112, 114, 105, 99, 101], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='\":', bytes=[34, 58], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32], logprob=-1.9361264946837764e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='250', bytes=[50, 53, 48], logprob=-3.128163257315464e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='.', bytes=[46], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='00', bytes=[48, 48], logprob=-1.8624639324116288e-06, top_logprobs=[]), ChatCompletionTokenLogprob(token=',\\n', bytes=[44, 10], logprob=-3.130576806142926e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token=' ', bytes=[32, 32, 32], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='summary', bytes=[115, 117, 109, 109, 97, 114, 121], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='\":', bytes=[34, 58], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token=' \"', bytes=[32, 34], logprob=-4.320199877838604e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='A', bytes=[65], logprob=-1.41616690158844, top_logprobs=[]), ChatCompletionTokenLogprob(token=' flexible', bytes=[32, 102, 108, 101, 120, 105, 98, 108, 101], logprob=-1.1143524646759033, top_logprobs=[]), ChatCompletionTokenLogprob(token=' and', bytes=[32, 97, 110, 100], logprob=-0.18483828008174896, top_logprobs=[]), ChatCompletionTokenLogprob(token=' elegant', bytes=[32, 101, 108, 101, 103, 97, 110, 116], logprob=-0.42920824885368347, top_logprobs=[]), ChatCompletionTokenLogprob(token=' table', bytes=[32, 116, 97, 98, 108, 101], logprob=-1.285778522491455, top_logprobs=[]), ChatCompletionTokenLogprob(token=' lamp', bytes=[32, 108, 97, 109, 112], logprob=-4.143808109802194e-05, top_logprobs=[]), ChatCompletionTokenLogprob(token=' designed', bytes=[32, 100, 101, 115, 105, 103, 110, 101, 100], logprob=-0.31241145730018616, top_logprobs=[]), ChatCompletionTokenLogprob(token=' by', bytes=[32, 98, 121], logprob=-0.12911923229694366, top_logprobs=[]), ChatCompletionTokenLogprob(token=' Maria', bytes=[32, 77, 97, 114, 105, 97], logprob=-0.07896524667739868, top_logprobs=[]), ChatCompletionTokenLogprob(token=' Ber', bytes=[32, 66, 101, 114], logprob=-4.320199877838604e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='nt', bytes=[110, 116], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='sen', bytes=[115, 101, 110], logprob=0.0, top_logprobs=[]), ChatCompletionTokenLogprob(token='.\"\\n', bytes=[46, 34, 10], logprob=-0.36576616764068604, top_logprobs=[]), ChatCompletionTokenLogprob(token='}\\n', bytes=[125, 10], logprob=-6.704273118884885e-07, top_logprobs=[]), ChatCompletionTokenLogprob(token='```', bytes=[96, 96, 96], logprob=-0.00861456897109747, top_logprobs=[])], refusal=None), message=ChatCompletionMessage(content='Analysis of Price:\\n\\nThe product description does not provide an explicit price for the Stay table lamp. To estimate the price, it\\'s important to consider factors such as the design origin (Danish design by Maria Berntsen), functionality (flexible swivel mechanism and adjustable arm and head), material quality (silky matte grey metal), and intended use (task lighting in modern, minimalistic settings).\\n\\nBased on these characteristics, the Stay table lamp is likely positioned as a mid-to-high-end product in the market. Danish-designed lighting products known for combining aesthetics with functionality often range from approximately $150 to $400. Given these aspects, a reasonable estimate for the price could be around $250.00.\\n\\nNow, I will provide the JSON dict with all extracted information:\\n\\n```json\\n{\\n \"materials\": [\"metal\"],\\n \"type\": \"table lamp\",\\n \"color\": \"silky matte grey\",\\n \"price\": 250.00,\\n \"summary\": \"A flexible and elegant table lamp designed by Maria Berntsen.\"\\n}\\n```', refusal=None, role='assistant', annotations=[], audio=None, function_call=None, tool_calls=None)), feature_weights=None, proba=None, score=None, weighted_spans=WeightedSpans(docs_weighted_spans=[DocWeightedSpans(document='Analysis of Price:\\n\\nThe product description does not provide an explicit price for the Stay table lamp. To estimate the price, it\\'s important to consider factors such as the design origin (Danish design by Maria Berntsen), functionality (flexible swivel mechanism and adjustable arm and head), material quality (silky matte grey metal), and intended use (task lighting in modern, minimalistic settings).\\n\\nBased on these characteristics, the Stay table lamp is likely positioned as a mid-to-high-end product in the market. Danish-designed lighting products known for combining aesthetics with functionality often range from approximately $150 to $400. Given these aspects, a reasonable estimate for the price could be around $250.00.\\n\\nNow, I will provide the JSON dict with all extracted information:\\n\\n```json\\n{\\n \"materials\": [\"metal\"],\\n \"type\": \"table lamp\",\\n \"color\": \"silky matte grey\",\\n \"price\": 250.00,\\n \"summary\": \"A flexible and elegant table lamp designed by Maria Berntsen.\"\\n}\\n```', spans=[('0-Analysis', [(0, 8)], 0.031100326941917205), ('8- of', [(8, 11)], 0.9272939728288944), ('11- Price', [(11, 17)], 0.05589812648675302), ('17-:\\n\\n', [(17, 20)], 0.6603840552609779), ('20-The', [(20, 23)], 0.8709716909780277), ('23- product', [(23, 31)], 0.908347115843313), ('31- description', [(31, 43)], 0.9473931442709769), ('43- does', [(43, 48)], 0.4834580811626623), ('48- not', [(48, 52)], 0.99999158136958), ('52- provide', [(52, 60)], 0.4124990598790427), ('60- an', [(60, 63)], 0.24490914328368124), ('63- explicit', [(63, 72)], 0.9838036964324991), ('72- price', [(72, 78)], 0.9965368507557492), ('78- for', [(78, 82)], 0.8671902953327638), ('82- the', [(82, 86)], 0.9998511778260051), ('86- Stay', [(86, 91)], 0.8120562338884505), ('91- table', [(91, 97)], 0.992260106545069), ('97- lamp', [(97, 102)], 1.0), ('102-.', [(102, 103)], 0.9541436290577758), ('103- To', [(103, 106)], 0.22682320353831265), ('106- estimate', [(106, 115)], 0.9253117057852979), ('115- the', [(115, 119)], 0.517016471439654), ('119- price', [(119, 125)], 0.9097433993978243), ('125-,', [(125, 126)], 0.983797236971635), (\"126- it's\", [(126, 131)], 0.008086241915920705), ('131- important', [(131, 141)], 0.5398335556628256), ('141- to', [(141, 144)], 0.9999647611318239), ('144- consider', [(144, 153)], 0.991330454602796), ('153- factors', [(153, 161)], 0.1633063958780871), ('161- such', [(161, 166)], 0.8704079321820062), ('166- as', [(166, 169)], 0.999999091165777), ('169- the', [(169, 173)], 0.6563860887178772), ('173- design', [(173, 180)], 0.4016105929444318), ('180- origin', [(180, 187)], 0.44259418878962736), ('187- (', [(187, 189)], 0.3741645797743778), ('189-D', [(189, 190)], 0.8931707985895362), ('190-anish', [(190, 195)], 0.9999995679801056), ('195- design', [(195, 202)], 0.5902483112299441), ('202- by', [(202, 205)], 0.8171392671637929), ('205- Maria', [(205, 211)], 0.9985591392345983), ('211- Ber', [(211, 215)], 0.9999984951481292), ('215-nt', [(215, 217)], 1.0), ('217-sen', [(217, 220)], 1.0), ('220-),', [(220, 222)], 0.9945350732042957), ('222- functionality', [(222, 236)], 0.05739063426730033), ('236- (', [(236, 238)], 0.6812345734502243), ('238-flex', [(238, 242)], 0.5065496545188568), ('242-ible', [(242, 246)], 0.9840781426664655), ('246- swivel', [(246, 253)], 0.9847790317898051), ('253- mechanism', [(253, 263)], 0.9979603114553249), ('263- and', [(263, 267)], 0.15329209193961424), ('267- adjustable', [(267, 278)], 0.9442024702453381), ('278- arm', [(278, 282)], 0.9241814800005089), ('282- and', [(282, 286)], 0.9376121333490623), ('286- head', [(286, 291)], 0.9968257886638747), ('291-),', [(291, 293)], 0.964875529680294), ('293- material', [(293, 302)], 0.25951460936867965), ('302- quality', [(302, 310)], 0.37236012322321166), ('310- (', [(310, 312)], 0.9972406616364523), ('312-sil', [(312, 315)], 0.9545252519700747), ('315-ky', [(315, 317)], 0.9990828888463521), ('317- matte', [(317, 323)], 0.9999857404571098), ('323- grey', [(323, 328)], 0.984854871756404), ('328- metal', [(328, 334)], 0.9997562021684266), ('334-),', [(334, 336)], 0.969349521440105), ('336- and', [(336, 340)], 0.9879049666617071), ('340- intended', [(340, 349)], 0.16653453221588588), ('349- use', [(349, 353)], 0.8940159982630072), ('353- (', [(353, 355)], 0.9795633227595859), ('355-task', [(355, 359)], 0.6490600978153632), ('359- lighting', [(359, 368)], 0.9999851444466235), ('368- in', [(368, 371)], 0.49860119061551494), ('371- modern', [(371, 378)], 0.06433551147059785), ('378-,', [(378, 379)], 0.17720624934739607), ('379- minimal', [(379, 387)], 0.8341657248077662), ('387-istic', [(387, 392)], 0.9976262179931533), ('392- settings', [(392, 401)], 0.7477839655600889), ('401-).\\n\\n', [(401, 405)], 0.1793207079905138), ('405-Based', [(405, 410)], 0.13044669418316154), ('410- on', [(410, 413)], 0.9999772771533194), ('413- these', [(413, 419)], 0.5172064905849382), ('419- characteristics', [(419, 435)], 0.13160568766887565), ('435-,', [(435, 436)], 0.72882064819143), ('436- the', [(436, 440)], 0.4111546577327526), ('440- Stay', [(440, 445)], 0.629192992357393), ('445- table', [(445, 451)], 0.9523976097795025), ('451- lamp', [(451, 456)], 0.999997779927489), ('456- is', [(456, 459)], 0.3177616774982452), ('459- likely', [(459, 466)], 0.8196736241452497), ('466- positioned', [(466, 477)], 0.8470375556685125), ('477- as', [(477, 480)], 0.5485250777340852), ('480- a', [(480, 482)], 0.9974892068693135), ('482- mid', [(482, 486)], 0.18107060060975305), ('486--to', [(486, 489)], 0.3496414586109216), ('489--high', [(489, 494)], 0.9713674967971203), ('494--end', [(494, 498)], 0.8306362578982314), ('498- product', [(498, 506)], 0.7956881109594603), ('506- in', [(506, 509)], 0.37530880993084603), ('509- the', [(509, 513)], 0.9612503965484102), ('513- market', [(513, 520)], 0.4096817278210386), ('520-.', [(520, 521)], 0.8640543479165882), ('521- Danish', [(521, 528)], 0.24037123784304548), ('528--designed', [(528, 537)], 0.24826218429718366), ('537- lighting', [(537, 546)], 0.7313082811139549), ('546- products', [(546, 555)], 0.4079146642145345), ('555- known', [(555, 561)], 0.0093925085990811), ('561- for', [(561, 565)], 0.9999883629029224), ('565- combining', [(565, 575)], 0.028738642745044892), ('575- aesthetics', [(575, 586)], 0.31613074303053984), ('586- with', [(586, 591)], 0.46711680701534064), ('591- functionality', [(591, 605)], 0.9683469375664772), ('605- often', [(605, 611)], 0.2908560811650651), ('611- range', [(611, 617)], 0.36132602383397133), ('617- from', [(617, 622)], 0.6216867820635172), ('622- approximately', [(622, 636)], 0.3481180300838724), ('636- $', [(636, 638)], 0.9742855470140944), ('638-150', [(638, 641)], 0.7137358769780064), ('641- to', [(641, 644)], 0.9461526301807588), ('644- $', [(644, 646)], 0.9950840181416029), ('646-400', [(646, 649)], 0.4152115349076142), ('649-.', [(649, 650)], 0.3851954516350021), ('650- Given', [(650, 656)], 0.6755684074178104), ('656- these', [(656, 662)], 0.049158122143082114), ('662- aspects', [(662, 670)], 0.031105725454522743), ('670-,', [(670, 671)], 0.796016632421059), ('671- a', [(671, 673)], 0.5778706122255274), ('673- reasonable', [(673, 684)], 0.9398450250520831), ('684- estimate', [(684, 693)], 0.5107392159521865), ('693- for', [(693, 697)], 0.967931183260857), ('697- the', [(697, 701)], 0.9396582688316502), ('701- price', [(701, 707)], 0.5786182497626335), ('707- could', [(707, 713)], 0.10029894280897976), ('713- be', [(713, 716)], 0.9825852239159907), ('716- around', [(716, 723)], 0.7904551041308469), ('723- $', [(723, 725)], 0.9781379488338885), ('725-250', [(725, 728)], 0.9213327394318271), ('728-.', [(728, 729)], 0.9785310222449798), ('729-00', [(729, 731)], 0.9977228045387296), ('731-.\\n\\n', [(731, 734)], 0.7759446721579865), ('734-Now', [(734, 737)], 0.3066629182800305), ('737-,', [(737, 738)], 0.9787496706186609), ('738- I', [(738, 740)], 0.2696778432668547), ('740- will', [(740, 745)], 0.9957069536427563), ('745- provide', [(745, 753)], 0.7586335120570286), ('753- the', [(753, 757)], 0.9754992165350655), ('757- JSON', [(757, 762)], 0.7969566591287495), ('762- dict', [(762, 767)], 0.2677579135800676), ('767- with', [(767, 772)], 0.9682210916996221), ('772- all', [(772, 776)], 0.0003982148526756783), ('776- extracted', [(776, 786)], 0.47392288610067923), ('786- information', [(786, 798)], 0.1360480803897047), ('798-:\\n\\n', [(798, 801)], 0.5321298610817511), ('801-```', [(801, 804)], 1.0), ('804-json', [(804, 808)], 1.0), ('808-\\n', [(808, 809)], 1.0), ('809-{\\n', [(809, 811)], 0.9999937270200764), ('811- ', [(811, 814)], 0.6224132144588476), ('814- \"', [(814, 816)], 1.0), ('816-materials', [(816, 825)], 0.9999995679801056), ('825-\":', [(825, 827)], 1.0), ('827- [\"', [(827, 830)], 0.9989667730739981), ('830-metal', [(830, 835)], 0.9722797716052907), ('835-\"],\\n', [(835, 839)], 0.9991270335127975), ('839- ', [(839, 842)], 1.0), ('842- \"', [(842, 844)], 1.0), ('844-type', [(844, 848)], 0.9999998063873693), ('848-\":', [(848, 850)], 0.9999996871837232), ('850- \"', [(850, 852)], 0.9056012682481982), ('852-table', [(852, 857)], 0.6771100347727493), ('857- lamp', [(857, 862)], 0.9940595274045081), ('862-\",\\n', [(862, 865)], 0.999653988585211), ('865- ', [(865, 868)], 1.0), ('868- \"', [(868, 870)], 1.0), ('870-color', [(870, 875)], 0.9999998063873693), ('875-\":', [(875, 877)], 0.9999998063873693), ('877- \"', [(877, 879)], 0.9999998063873693), ('879-sil', [(879, 882)], 0.7962060486763332), ('882-ky', [(882, 884)], 0.9997025970321683), ('884- matte', [(884, 890)], 0.999999091165777), ('890- grey', [(890, 895)], 0.999997303114013), ('895-\",\\n', [(895, 898)], 0.9999907469519438), ('898- ', [(898, 901)], 0.9999998063873693), ('901- \"', [(901, 903)], 1.0), ('903-price', [(903, 908)], 0.9999998063873693), ('908-\":', [(908, 910)], 1.0), ('910- ', [(910, 911)], 0.9999998063873693), ('911-250', [(911, 914)], 0.9999996871837232), ('914-.', [(914, 915)], 1.0), ('915-00', [(915, 917)], 0.999998137537802), ('917-,\\n', [(917, 919)], 0.999968694721959), ('919- ', [(919, 922)], 1.0), ('922- \"', [(922, 924)], 1.0), ('924-summary', [(924, 931)], 1.0), ('931-\":', [(931, 933)], 1.0), ('933- \"', [(933, 935)], 0.9999995679801056), ('935-A', [(935, 936)], 0.24264230849118593), ('936- flexible', [(936, 945)], 0.3281276843885938), ('945- and', [(945, 949)], 0.8312387008376273), ('949- elegant', [(949, 957)], 0.6510243399908526), ('957- table', [(957, 963)], 0.27643528874717094), ('963- lamp', [(963, 968)], 0.9999585627774474), ('968- designed', [(968, 977)], 0.7316804110376031), ('977- by', [(977, 980)], 0.8788691697083658), ('980- Maria', [(980, 986)], 0.9240720384607087), ('986- Ber', [(986, 990)], 0.9999995679801056), ('990-nt', [(990, 992)], 1.0), ('992-sen', [(992, 995)], 1.0), ('995-.\"\\n', [(995, 998)], 0.6936649835669562), ('998-}\\n', [(998, 1000)], 0.9999993295729128), ('1000-```', [(1000, 1003)], 0.991422430108248)], preserve_density=False, with_probabilities=True, vec_name=None)], other=None), heatmap=None)], feature_importances=None, decision_tree=None, highlight_spaces=None, transition_features=None, image=None)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "eli5.explain_prediction(client, prompt_cot, model='gpt-4o')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 4e5a3279012d0d78e1c1ed3c8032b48d0f6e5b02 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Sat, 5 Apr 2025 19:55:17 +0100 Subject: [PATCH 09/12] fix typecheck, include openai dependency into tox config --- eli5/base.py | 4 ++-- eli5/llm/explain_prediction.py | 10 ++++++++-- tests/test_llm_explain_prediction.py | 9 +++++---- tox.ini | 3 +++ 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/eli5/base.py b/eli5/base.py index 46f41183..f10293ad 100644 --- a/eli5/base.py +++ b/eli5/base.py @@ -1,4 +1,4 @@ -from typing import Union, Optional +from typing import Union, Optional, Sequence import numpy as np @@ -152,7 +152,7 @@ class DocWeightedSpans: """ def __init__(self, document: str, - spans: list[WeightedSpan], + spans: Sequence[WeightedSpan], preserve_density: Optional[bool] = None, with_probabilities: Optional[bool] = None, vec_name: Optional[str] = None, diff --git a/eli5/llm/explain_prediction.py b/eli5/llm/explain_prediction.py index 0af714b3..e0a9d416 100644 --- a/eli5/llm/explain_prediction.py +++ b/eli5/llm/explain_prediction.py @@ -19,6 +19,8 @@ def explain_prediction_openai_logprobs(logprobs: ChoiceLogprobs, doc=None): while unlikely tokens are highlighted in red. ``doc`` argument is ignored. """ + if logprobs.content is None: + raise ValueError('Predictions must be obtained with logprobs enabled') text = ''.join(x.token for x in logprobs.content) spans = [] idx = 0 @@ -46,7 +48,7 @@ def explain_prediction_openai_logprobs(logprobs: ChoiceLogprobs, doc=None): @explain_prediction.register(ChatCompletion) def explain_prediction_openai_completion( - chat_completion: ChoiceLogprobs, doc=None): + chat_completion: ChatCompletion, doc=None): """ Creates an explanation of the ChatCompletion's logprobs highlighting them proportionally to the log probability. More likely tokens are highligted in green, @@ -55,6 +57,8 @@ def explain_prediction_openai_completion( """ targets = [] for choice in chat_completion.choices: + if choice.logprobs is None: + raise ValueError('Predictions must be obtained with logprobs enabled') target, = explain_prediction_openai_logprobs(choice.logprobs).targets target.target = choice targets.append(target) @@ -88,5 +92,7 @@ def explain_prediction_openai_client( messages = doc kwargs['logprobs'] = True chat_completion = client.chat.completions.create( - messages=messages, model=model, **kwargs) + messages=messages, # type: ignore + model=model, + **kwargs) return explain_prediction_openai_completion(chat_completion) diff --git a/tests/test_llm_explain_prediction.py b/tests/test_llm_explain_prediction.py index df241ef9..a8faaee7 100644 --- a/tests/test_llm_explain_prediction.py +++ b/tests/test_llm_explain_prediction.py @@ -2,10 +2,7 @@ import pytest from unittest.mock import Mock -import eli5 -from eli5.base import Explanation -from eli5.formatters.html import format_as_html - +pytest.importorskip('openai') from openai.types.chat.chat_completion import ( ChoiceLogprobs, ChatCompletion, @@ -15,6 +12,10 @@ ) from openai import Client +import eli5 +from eli5.base import Explanation +from eli5.formatters.html import format_as_html + @pytest.fixture def example_logprobs(): diff --git a/tox.ini b/tox.ini index d8893a98..d0717246 100644 --- a/tox.ini +++ b/tox.ini @@ -29,6 +29,7 @@ deps= ipython pandas sklearn-crfsuite + openai commands= pip install -e . py.test --doctest-modules \ @@ -87,6 +88,8 @@ commands= --ignore eli5/keras \ --ignore eli5/formatters/as_dataframe.py \ --ignore eli5/formatters/image.py \ + --ignore eli5/formatters/image.py \ + --ignore eli5/llm/explain_prediction.py \ --ignore tests/utils_image.py \ --cov=eli5 --cov-report=html --cov-report=term {posargs: eli5 tests} From 0ff63df6e21e0a2e6ccac841905c37d349600dba Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Sat, 5 Apr 2025 20:14:43 +0100 Subject: [PATCH 10/12] fix python 3.9 compatibility --- eli5/llm/explain_prediction.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eli5/llm/explain_prediction.py b/eli5/llm/explain_prediction.py index e0a9d416..f11680a4 100644 --- a/eli5/llm/explain_prediction.py +++ b/eli5/llm/explain_prediction.py @@ -1,4 +1,5 @@ import math +from typing import Union import openai from openai.types.chat.chat_completion import ChoiceLogprobs, ChatCompletion @@ -72,7 +73,7 @@ def explain_prediction_openai_completion( @explain_prediction.register(openai.Client) def explain_prediction_openai_client( client: openai.Client, - doc: str | list[dict], + doc: Union[str, list[dict]], *, model: str, **kwargs, From 81fd448e27e7bd5d0eb9a5354243ee9a8a3e02c1 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Sat, 5 Apr 2025 20:25:07 +0100 Subject: [PATCH 11/12] Fix typos Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- eli5/llm/explain_prediction.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eli5/llm/explain_prediction.py b/eli5/llm/explain_prediction.py index f11680a4..49470c12 100644 --- a/eli5/llm/explain_prediction.py +++ b/eli5/llm/explain_prediction.py @@ -16,7 +16,7 @@ def explain_prediction_openai_logprobs(logprobs: ChoiceLogprobs, doc=None): """ Creates an explanation of the logprobs (available as ``.choices[idx].logprobs`` on a ChatCompletion object), highlighting them proportionally to the log probability. - More likely tokens are highligted in green, + More likely tokens are highlighted in green, while unlikely tokens are highlighted in red. ``doc`` argument is ignored. """ @@ -52,7 +52,7 @@ def explain_prediction_openai_completion( chat_completion: ChatCompletion, doc=None): """ Creates an explanation of the ChatCompletion's logprobs highlighting them proportionally to the log probability. - More likely tokens are highligted in green, + More likely tokens are highlighted in green, while unlikely tokens are highlighted in red. ``doc`` argument is ignored. """ From 7b9dd933b1ff5f662d631ab0c67186cc810d1156 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Sat, 5 Apr 2025 20:27:09 +0100 Subject: [PATCH 12/12] fix more typos --- docs/source/libraries/openai.rst | 2 +- eli5/llm/explain_prediction.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/libraries/openai.rst b/docs/source/libraries/openai.rst index 63af0f99..7c87e1d9 100644 --- a/docs/source/libraries/openai.rst +++ b/docs/source/libraries/openai.rst @@ -11,7 +11,7 @@ eli5 supports :func:`eli5.explain_prediction` for ``ChatCompletion``, ``ChoiceLogprobs`` and ``openai.Client`` objects, highlighting tokens proportionally to the log probability, which can help to see where model is less confident in it's predictions. -More likely tokens are highligted in green, +More likely tokens are highlighted in green, while unlikely tokens are highlighted in red: .. image:: ../static/llm-explain-logprobs.png diff --git a/eli5/llm/explain_prediction.py b/eli5/llm/explain_prediction.py index 49470c12..8d078be0 100644 --- a/eli5/llm/explain_prediction.py +++ b/eli5/llm/explain_prediction.py @@ -82,7 +82,7 @@ def explain_prediction_openai_client( Calls OpenAI client, obtaining response for ``doc`` (a string, or a list of messages), with logprobs enabled, and explains the prediction, highlighting tokens proportionally to the log probability. - More likely tokens are highligted in green, + More likely tokens are highlighted in green, while unlikely tokens are highlighted in red. . Other keyword arguments are passed to OpenAI client, with ``model`` keyword argument required.