adding support for Palm, Cohere, HF Inference Endpoints, Replicate

README.md

@@ -259,7 +259,7 @@ Note if you want to remotely connect to a RealChar server, SSL set up is require
 
 <br/>
 
-## 🆕! Anyscale and LangSmith integration
+## 🆕! Anyscale, LangSmith, LiteLLM integrations
 <details><summary>👇click me</summary>
 
 ### Anyscale
@@ -279,6 +279,16 @@ LANGCHAIN_PROJECT=YOUR_LANGCHAIN_PROJECT
 ```
 And it should work out of the box.
 
+
+### LiteLLM
+Use any model from Replicate, Cohere, AI21, Huggingface, TogetherAI, Aleph Alpha, and more - https://docs.litellm.ai/docs/completion/supported
+
+in your .env, just set the `LITELLM_API_KEY` to be the one for any of the providers you want to use
+```
+LITELLM_API_KEY="" # this is the api key for any of the providers supported by litellm
+```
+
+and this will be passed to litellm to use! 
 </details>
 
 <br/>

realtime_ai_character/llm/__init__.py

@@ -1,13 +1,8 @@
-import os
-from functools import cache
+from realtime_ai_character.llm.base import AsyncCallbackAudioHandler, AsyncCallbackTextHandler, LLM
+import litellm
 
-from langchain.chat_models.base import BaseChatModel
-
-from realtime_ai_character.llm.base import LLM
-
-
-def get_llm(model="gpt-3.5-turbo-16k") -> LLM:
-    if model.startswith('gpt'):
+def get_llm(model='gpt-3.5-turbo-16k') -> LLM:
+    if model.startswith('gpt') or model in litellm.model_list or model.split("/")[0] in litellm.provider_list:
         from realtime_ai_character.llm.openai_llm import OpenaiLlm
         return OpenaiLlm(model=model)
     elif model.startswith('claude'):

realtime_ai_character/llm/openai_llm.py

@@ -5,7 +5,7 @@
 if os.getenv('OPENAI_API_TYPE') == 'azure':
     from langchain.chat_models import AzureChatOpenAI
 else:
-    from langchain.chat_models import ChatOpenAI
+    from langchain.chat_models import ChatLiteLLM
 from langchain.schema import BaseMessage, HumanMessage
 
 from realtime_ai_character.database.chroma import get_chroma
@@ -28,10 +28,11 @@ def __init__(self, model):
                 streaming=True
             )
         else:
-            self.chat_open_ai = ChatOpenAI(
+            self.chat_open_ai = ChatLiteLLM(
                 model=model,
                 temperature=0.5,
-                streaming=True
+                streaming=True,
+                api_key=os.getenv("LITELLM_API_KEY")
             )
         self.config = {
             "model": model,

requirements.txt

@@ -38,3 +38,4 @@ typing
 uvicorn
 websockets
 google-auth
+litellm