Skip to content

FEATURE: adding ability to add system messages #21

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
maishathasin wants to merge 7 commits into
base: main
Choose a base branch
from
Open

FEATURE: adding ability to add system messages #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
9b87024
add system messages
maishathasin Apr 30, 2024
074c7e2
rmv examples
maishathasin Apr 30, 2024
5bd5527
Update examples.ipynb
maishathasin Apr 30, 2024
5b291e1
Update examples.ipynb
maishathasin Apr 30, 2024
0f0b831
rmv examples
maishathasin Apr 30, 2024
2f13891
Merge branch 'main' of https://github.com/maishathasin/doctran
maishathasin Apr 30, 2024
c9f47ea
add optional system message
maishathasin May 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions doctran/doctran.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ class Document(BaseModel):
content_type: ContentType
raw_content: str
transformed_content: str
system: Optional[str]
config: DoctranConfig
extracted_properties: Optional[Dict] = {}
metadata: Optional[Dict[str, Any]] = None
Expand Down Expand Up @@ -193,7 +194,7 @@ def __init__(self, openai_api_key: str = None, openai_model: str = "gpt-4", open
if os.environ.get('OPENAI_API_VERSION'):
self.config.openai.api_version = os.environ['OPENAI_API_VERSION']

def parse(self, *, content: str, content_type: ContentType = "text", uri: str = None, metadata: dict = None) -> Document:
def parse(self, *, content: str,system: Optional[str] = None, content_type: ContentType = "text", uri: str = None, metadata: dict = None) -> Document:
'''
Parse raw text and apply different chunking schemes based on the content type.

Expand All @@ -204,5 +205,5 @@ def parse(self, *, content: str, content_type: ContentType = "text", uri: str =
uri = str(uuid.uuid4())
if content_type == ContentType.text.value:
# TODO: Optional chunking for documents that are too large
document = Document(id=str(uuid.uuid4()), content_type=content_type, raw_content=content, transformed_content=content, config=self.config, uri=uri, metadata=metadata)
document = Document(id=str(uuid.uuid4()), content_type=content_type, raw_content=content, transformed_content=content,system=system, config=self.config, uri=uri, metadata=metadata)
return document
5 changes: 4 additions & 1 deletion doctran/transformers/transformers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import tiktoken
from doctran import Document, DoctranConfig, ExtractProperty, RecognizerEntity


class TooManyTokensException(Exception):
def __init__(self, content_token_size: int, token_limit: int):
super().__init__(f"OpenAI document transformation failed. The document is {content_token_size} tokens long, which exceeds the token limit of {token_limit}.")
Expand Down Expand Up @@ -59,7 +60,9 @@ def executeOpenAICall(self, document: Document) -> Document:
function_call = OpenAIFunctionCall(
seed=self.config.openai_deployment_id,
model=self.config.openai_model,
messages=[{"role": "user", "content": document.transformed_content}],
messages=[{"role": "system", "content":document.system},

{"role": "user", "content": document.transformed_content}],
tools=[{
"type": "function",
"function": {
Expand Down