feat: implement multimodal support with user-defined overrides and remove screenshot functionality

Author: gary149

.gitignore

@@ -14,4 +14,5 @@ SECRET_CONFIG
 gcp-*.json
 db
 models/*
-!models/add-your-models-here.txt
+!models/add-your-models-here.txt
+scirac

scira-chat

@@ -0,0 +1 @@
+Subproject commit 892ec4a60d23cd1c19b1a079c799d24a8a79a262

src/lib/components/Switch.svelte

@@ -1,33 +1,33 @@
 <script lang="ts">
-    interface Props {
-        checked: boolean;
-        name: string;
-    }
+	interface Props {
+		checked: boolean;
+		name: string;
+	}
 
-    let { checked = $bindable(), name }: Props = $props();
+	let { checked = $bindable(), name }: Props = $props();
 
-    function toggle() {
-        checked = !checked;
-    }
+	function toggle() {
+		checked = !checked;
+	}
 
-    function onKeydown(e: KeyboardEvent) {
-        if (e.key === " " || e.key === "Enter") {
-            e.preventDefault();
-            toggle();
-        }
-    }
+	function onKeydown(e: KeyboardEvent) {
+		if (e.key === " " || e.key === "Enter") {
+			e.preventDefault();
+			toggle();
+		}
+	}
 </script>
 
 <input bind:checked type="checkbox" {name} class="peer pointer-events-none absolute opacity-0" />
 <div
-    aria-checked={checked}
-    aria-roledescription="switch"
-    aria-label="switch"
-    role="switch"
-    tabindex="0"
-    onclick={toggle}
-    onkeydown={onKeydown}
-    class="relative inline-flex h-5 w-9 shrink-0 cursor-pointer items-center rounded-full bg-gray-300 p-1 shadow-inner ring-gray-400 transition-all hover:bg-gray-400 focus-visible:ring focus-visible:ring-offset-1 peer-checked:bg-blue-600 dark:bg-gray-600 peer-checked:[&>div]:translate-x-3.5"
+	aria-checked={checked}
+	aria-roledescription="switch"
+	aria-label="switch"
+	role="switch"
+	tabindex="0"
+	onclick={toggle}
+	onkeydown={onKeydown}
+	class="relative inline-flex h-5 w-9 shrink-0 cursor-pointer items-center rounded-full bg-gray-300 p-1 shadow-inner ring-gray-400 transition-all peer-checked:bg-black hover:bg-gray-400 focus-visible:ring focus-visible:ring-offset-1 peer-checked:[&>div]:translate-x-3.5"
 >
-    <div class="h-3.5 w-3.5 rounded-full bg-white shadow-sm transition-all"></div>
-    </div>
+	<div class="h-3.5 w-3.5 rounded-full bg-white shadow-sm transition-all"></div>
+</div>

src/lib/components/chat/ChatInput.svelte

@@ -5,8 +5,6 @@
 	import IconPaperclip from "$lib/components/icons/IconPaperclip.svelte";
 	import { useSettingsStore } from "$lib/stores/settings";
 	import { page } from "$app/state";
-	import { captureScreen } from "$lib/utils/screenshot";
-	import IconScreenshot from "../icons/IconScreenshot.svelte";
 	import { loginModalOpen } from "$lib/stores/loginModal";
 
 	import { isVirtualKeyboard } from "$lib/utils/isVirtualKeyboard";
@@ -94,7 +92,7 @@
 
 	const settings = useSettingsStore();
 
-	// Tools removed; only show file upload and screenshot when applicable
+	// Tools removed; only show file upload when applicable
 	let showFileUpload = $derived(modelIsMultimodal && mimeTypes.length > 0);
 	let showNoTools = $derived(!showFileUpload);
 </script>
@@ -162,32 +160,7 @@
 						</label>
 					</HoverTooltip>
 				</div>
-				{#if mimeTypes.includes("image/*")}
-					<HoverTooltip
-						label="Capture screenshot"
-						position="top"
-						TooltipClassNames="text-xs !text-left !w-auto whitespace-nowrap !py-1 !mb-0 max-sm:hidden"
-					>
-						<button
-							class="base-tool"
-							onclick={async (e) => {
-								e.preventDefault();
-								const screenshot = await captureScreen();
-
-								// Convert base64 to blob
-								const base64Response = await fetch(screenshot);
-								const blob = await base64Response.blob();
-
-								// Create a File object from the blob
-								const file = new File([blob], "screenshot.png", { type: "image/png" });
-
-								files = [...files, file];
-							}}
-						>
-							<IconScreenshot classNames="text-xl" />
-						</button>
-					</HoverTooltip>
-				{/if}
+
 			{/if}
 		</div>
 	{/if}

src/lib/components/chat/ChatWindow.svelte

@@ -178,10 +178,14 @@
 
 	const settings = useSettingsStore();
 
+	// Respect per‑model multimodal toggle from settings (force enable)
+	let modelIsMultimodal = $derived(
+		currentModel.multimodal || ($settings.multimodalOverrides?.[currentModel.id] ?? false)
+	);
 	let activeMimeTypes = $derived(
 		Array.from(
 			new Set([
-				...(currentModel.multimodal
+				...(modelIsMultimodal
 					? (currentModel.multimodalAcceptedMimetypes ?? ["image/*"])
 					: []),
 			])
@@ -364,7 +368,7 @@
 								on:submit={handleSubmit}
 								{onPaste}
 								disabled={isReadOnly || lastIsError}
-								modelIsMultimodal={currentModel.multimodal}
+								modelIsMultimodal={modelIsMultimodal}
 								bind:focused
 							/>
 						{/if}

src/lib/components/icons/IconScreenshot.svelte

@@ -72,6 +72,7 @@ export const userGroup = new Elysia()
 						DEFAULT_SETTINGS.shareConversationsWithModelAuthors,
 
 					customPrompts: settings?.customPrompts ?? {},
+					multimodalOverrides: settings?.multimodalOverrides ?? {},
 				};
 			})
 			.post("/settings", async ({ locals, request }) => {
@@ -86,6 +87,7 @@ export const userGroup = new Elysia()
 						ethicsModalAccepted: z.boolean().optional(),
 						activeModel: z.string().default(DEFAULT_SETTINGS.activeModel),
 						customPrompts: z.record(z.string()).default({}),
+						multimodalOverrides: z.record(z.boolean()).default({}),
 						disableStream: z.boolean().default(false),
 						directPaste: z.boolean().default(false),
 					})

src/lib/server/api/routes/groups/user.ts

@@ -31,24 +31,21 @@ export const endpointOAIParametersSchema = z.object({
 	defaultHeaders: z.record(z.string()).optional(),
 	defaultQuery: z.record(z.string()).optional(),
 	extraBody: z.record(z.any()).optional(),
-	multimodal: z
-		.object({
-			image: createImageProcessorOptionsValidator({
-				supportedMimeTypes: [
-					"image/png",
-					"image/jpeg",
-					"image/webp",
-					"image/avif",
-					"image/tiff",
-					"image/gif",
-				],
-				preferredMimeType: "image/webp",
-				maxSizeInMB: Infinity,
-				maxWidth: 4096,
-				maxHeight: 4096,
-			}),
-		})
-		.default({}),
+    multimodal: z
+        .object({
+            image: createImageProcessorOptionsValidator({
+                supportedMimeTypes: [
+                    // Restrict to the most widely-supported formats
+                    "image/png",
+                    "image/jpeg",
+                ],
+                preferredMimeType: "image/jpeg",
+                maxSizeInMB: 3,
+                maxWidth: 2048,
+                maxHeight: 2048,
+            }),
+        })
+        .default({}),
 	/* enable use of max_completion_tokens in place of max_tokens */
 	useCompletionTokens: z.boolean().default(false),
 	streamingSupported: z.boolean().default(true),
@@ -118,11 +115,15 @@ export async function endpointOai(
 
 			return openAICompletionToTextGenerationStream(openAICompletion);
 		};
-	} else if (completion === "chat_completions") {
-		return async ({ messages, preprompt, generateSettings, conversationId }) => {
+    } else if (completion === "chat_completions") {
+        return async ({ messages, preprompt, generateSettings, conversationId, isMultimodal }) => {
 			// Format messages for the chat API, handling multimodal content if supported
-			let messagesOpenAI: OpenAI.Chat.Completions.ChatCompletionMessageParam[] =
-				await prepareMessages(messages, imageProcessor, model.multimodal);
+            let messagesOpenAI: OpenAI.Chat.Completions.ChatCompletionMessageParam[] =
+                await prepareMessages(
+                    messages,
+                    imageProcessor,
+                    isMultimodal ?? model.multimodal
+                );
 
 			// Check if a system message already exists as the first message
 			const hasSystemMessage = messagesOpenAI.length > 0 && messagesOpenAI[0]?.role === "system";
@@ -214,36 +215,34 @@ async function prepareMessages(
 	imageProcessor: ReturnType<typeof makeImageProcessor>,
 	isMultimodal: boolean
 ): Promise<OpenAI.Chat.Completions.ChatCompletionMessageParam[]> {
-	return Promise.all(
-		messages.map(async (message) => {
-			if (message.from === "user" && isMultimodal) {
-				return {
-					role: message.from,
-					content: [
-						...(await prepareFiles(imageProcessor, message.files ?? [])),
-						{ type: "text", text: message.content },
-					],
-				};
-			}
-			return {
-				role: message.from,
-				content: message.content,
-			};
-		})
-	);
+    return Promise.all(
+        messages.map(async (message) => {
+            if (message.from === "user" && isMultimodal) {
+                const parts = [
+                    { type: "text" as const, text: message.content },
+                    ...(await prepareFiles(imageProcessor, message.files ?? [])),
+                ];
+                return { role: message.from, content: parts };
+            }
+            return { role: message.from, content: message.content };
+        })
+    );
 }
 
 async function prepareFiles(
-	imageProcessor: ReturnType<typeof makeImageProcessor>,
-	files: MessageFile[]
+    imageProcessor: ReturnType<typeof makeImageProcessor>,
+    files: MessageFile[]
 ): Promise<OpenAI.Chat.Completions.ChatCompletionContentPartImage[]> {
-	const processedFiles = await Promise.all(
-		files.filter((file) => file.mime.startsWith("image/")).map(imageProcessor)
-	);
-	return processedFiles.map((file) => ({
-		type: "image_url" as const,
-		image_url: {
-			url: `data:${file.mime};base64,${file.image.toString("base64")}`,
-		},
-	}));
+    const processedFiles = await Promise.all(
+        files.filter((file) => file.mime.startsWith("image/")).map(imageProcessor)
+    );
+    return processedFiles.map((file) => ({
+        type: "image_url" as const,
+        image_url: {
+            url: `data:${file.mime};base64,${file.image.toString("base64")}`,
+            // Improves compatibility with some OpenAI-compatible servers
+            // that expect an explicit detail setting.
+            detail: "auto",
+        },
+    }));
 }

src/lib/server/endpoints/openai/endpointOai.ts

@@ -14,7 +14,16 @@ import { logger } from "../logger";
 type GenerateContext = Omit<TextGenerationContext, "messages"> & { messages: EndpointMessage[] };
 
 export async function* generate(
-	{ model, endpoint, conv, messages, assistant, isContinue, promptedAt }: GenerateContext,
+	{
+		model,
+		endpoint,
+		conv,
+		messages,
+		assistant,
+		isContinue,
+		promptedAt,
+		forceMultimodal,
+	}: GenerateContext,
 	preprompt?: string
 ): AsyncIterable<MessageUpdate> {
 	// reasoning mode is false by default
@@ -45,7 +54,8 @@ export async function* generate(
 		preprompt,
 		continueMessage: isContinue,
 		generateSettings: assistant?.generateSettings,
-		isMultimodal: model.multimodal,
+		// Allow user-level override to force multimodal
+		isMultimodal: (forceMultimodal ?? false) || model.multimodal,
 		conversationId: conv._id,
 	})) {
 		// text generation completed

src/lib/server/textGeneration/generate.ts

@@ -5,13 +5,15 @@ import type { Message } from "$lib/types/Message";
 import type { Assistant } from "$lib/types/Assistant";
 
 export interface TextGenerationContext {
-	model: ProcessedModel;
-	endpoint: Endpoint;
-	conv: Conversation;
-	messages: Message[];
-	assistant?: Pick<Assistant, "dynamicPrompt" | "generateSettings">;
-	isContinue: boolean;
-	promptedAt: Date;
-	ip: string;
-	username?: string;
+    model: ProcessedModel;
+    endpoint: Endpoint;
+    conv: Conversation;
+    messages: Message[];
+    assistant?: Pick<Assistant, "dynamicPrompt" | "generateSettings">;
+    isContinue: boolean;
+    promptedAt: Date;
+    ip: string;
+    username?: string;
+    /** Force-enable multimodal handling for endpoints that support it */
+    forceMultimodal?: boolean;
 }