Merge pull request #129 from fixie-ai/juberti/baseten2

Update baseten to v0.4

Author: juberti

llm_benchmark_suite.py

@@ -492,9 +492,9 @@ def _audio_models():
         _UltravoxLlm("fixie-ai/ultravox-v0.4", "ultravox-v0.4-8b"),
         _UltravoxLlm("fixie-ai/ultravox-70B", "ultravox-v0.4-70b"),
         _Llm(
-            "fixie-ai/ultravox-v0.2",
-            "baseten.co/ultravox-v0.2",
-            base_url="https://bridge.baseten.co/5wovovzq/v1/direct",
+            "ultravox",
+            "baseten.co/ultravox-v0.4",
+            base_url="https://bridge.baseten.co/v1/direct",
             api_key=os.getenv("BASETEN_API_KEY"),
         ),
     ]

llm_request.py

@@ -319,6 +319,9 @@ async def openai_chat(ctx: ApiContext, path: str = "/chat/completions") -> ApiRe
     # See https://github.com/Azure/azure-rest-api-specs/issues/25062
     if not any(p in ctx.name for p in ["azure", "databricks", "fireworks", "mistral"]):
         kwargs["stream_options"] = {"include_usage": True}
+    # Hack to identify our baseten deployment, which isn't contained in the URL.
+    if ctx.name.startswith("baseten"):
+        kwargs["baseten"] = {"model_id": "rwn2v41w"}
     data = make_openai_chat_body(ctx, **kwargs)
     return await post(ctx, url, headers, data, openai_chunk_gen)