Add comprehensive tests for routes and update CI workflow

.github/workflows/docker-publish.yml

@@ -21,6 +21,19 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@v1
+        with:
+          bun-version: latest
+
+      - name: Install dependencies
+        run: bun install
+
+      - name: Run tests
+        env:
+          DATABASE_PATH: ":memory:"
+        run: bun test
+
       - name: Log in to Docker Hub
         uses: docker/login-action@v3
         with:

.gitignore

@@ -33,3 +33,7 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
 
 # Finder (MacOS) folder config
 .DS_Store
+
+# SQLite database files
+*.sqlite
+*.sqlite-*

package.json

@@ -6,7 +6,7 @@
   "scripts": {
     "dev": "PORT=8000 bun run --watch src/server.ts",
     "start": "bun run ./src/database/migrations.ts up && bun run src/server.ts",
-    "test": "bun test"
+    "test": "bun run tests/run.ts"
   },
   "devDependencies": {
     "@types/bun": "latest",

tests/documentRoute.test.ts

@@ -0,0 +1,154 @@
+import {
+  expect,
+  describe,
+  test,
+  beforeAll,
+  beforeEach,
+  afterAll,
+  mock,
+} from "bun:test";
+
+// We need to set environment variables before importing the database module
+process.env.DATABASE_PATH = "******";
+process.env.AI_API_KEY = "test-key";
+process.env.AI_EMBEDDING_MODEL = "test-model";
+
+// Mock for generateEmbeddings
+const mockEmbeddings = Array(1536).fill(0.1);
+const generateEmbeddingsMock = mock(async (text, config) => {
+  // Return mock embeddings
+  return mockEmbeddings;
+});
+
+// Mock the generateEmbeddings module
+mock.module("../src/shared/generateEmbeddings", () => {
+  return {
+    generateEmbeddings: generateEmbeddingsMock,
+  };
+});
+
+// Import routes after mocking
+import { indexRoute } from "../src/routes/index";
+import { documentRoute } from "../src/routes/document/document";
+
+// Now we can import database and migrations
+import { db } from "../src/database/database";
+import { migrations } from "../src/database/migrations";
+
+describe("Document Route", () => {
+  // Set up the test environment and insert test document
+  let documentId: number;
+
+  beforeAll(() => {
+    // Run the migrations to create database schema
+    for (const migration of migrations) {
+      migration.up();
+    }
+  });
+
+  beforeEach(async () => {
+    // Clean up test data
+    db.exec("DELETE FROM document_chunks");
+    db.exec("DELETE FROM documents");
+
+    // Insert a test document to retrieve later
+    const sampleText = "This is a test document for document endpoint";
+    const sampleSource = "test-source";
+
+    // Create a request with sample data
+    const request = new Request("http://localhost/index", {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        source: sampleSource,
+        text: sampleText,
+        metadata: { testKey: "testValue" }
+      }),
+    });
+
+    // Process the request
+    await indexRoute(request);
+
+    // Get the document id
+    const document = db.query("SELECT id FROM documents LIMIT 1").get() as { id: number };
+    documentId = document.id;
+  });
+
+  // Close the database after all tests
+  afterAll(() => {
+    db.close();
+  });
+
+  test("should retrieve document by id", async () => {
+    // Create a request to retrieve the document
+    const request = new Request("http://localhost/document", {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        id: documentId,
+      }),
+    });
+
+    // Process the request
+    const response = await documentRoute(request);
+    const responseData = await response.json();
+
+    // Verify response
+    expect(response.status).toBe(200);
+    expect(responseData).toHaveProperty("document");
+
+    // Verify document properties
+    const document = responseData.document;
+    expect(document).toHaveProperty("id", documentId);
+    expect(document).toHaveProperty("text", "This is a test document for document endpoint");
+    expect(document).toHaveProperty("source", "test-source");
+    expect(document).toHaveProperty("metadata");
+    expect(document.metadata).toHaveProperty("testKey", "testValue");
+  });
+
+  test("should handle non-existent document id", async () => {
+    // Create a request with non-existent document id
+    const request = new Request("http://localhost/document", {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        id: 9999, // Non-existent ID
+      }),
+    });
+
+    // Process the request
+    const response = await documentRoute(request);
+    const responseData = await response.json();
+
+    // Verify error response
+    expect(response.status).toBe(200);
+    expect(responseData).toHaveProperty("error", "Document not found");
+    expect(responseData).not.toHaveProperty("document");
+  });
+
+  test("should handle invalid request with missing id", async () => {
+    // Create a request without an id
+    const request = new Request("http://localhost/document", {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({}),
+    });
+
+    // Process the request
+    const response = await documentRoute(request);
+    const responseData = await response.json();
+
+    // Verify validation error
+    expect(response.status).toBe(400);
+    expect(responseData).toHaveProperty("error", "Validation failed");
+    expect(responseData).toHaveProperty("issues");
+  });
+});

tests/indexRoute.test.ts

@@ -0,0 +1,119 @@
+import {
+  expect,
+  describe,
+  test,
+  beforeAll,
+  beforeEach,
+  afterAll,
+  mock,
+} from "bun:test";
+
+// We need to set environment variables before importing the database module
+process.env.DATABASE_PATH = ":memory:";
+process.env.AI_API_KEY = "test-key";
+process.env.AI_EMBEDDING_MODEL = "test-model";
+
+// Mock for generateEmbeddings
+const mockEmbeddings = Array(1536).fill(0.1);
+const generateEmbeddingsMock = mock(async (text, config) => {
+  // Return mock embeddings
+  return mockEmbeddings;
+});
+
+// Mock the generateEmbeddings module
+mock.module("../src/shared/generateEmbeddings", () => {
+  return {
+    generateEmbeddings: generateEmbeddingsMock,
+  };
+});
+
+// Import the indexRoute after mocking
+import { indexRoute } from "../src/routes/index";
+
+// Now we can import database and migrations
+import { db } from "../src/database/database";
+import { migrations } from "../src/database/migrations";
+
+describe("Index Route", () => {
+  // Set up the test environment
+  beforeAll(() => {
+    // Create migrations table
+    db.exec(`
+      CREATE TABLE IF NOT EXISTS migrations (
+        id SERIAL PRIMARY KEY,
+        name TEXT UNIQUE NOT NULL,
+        applied_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+      );
+    `);
+
+    // Run the migrations to create database schema
+    for (const migration of migrations) {
+      migration.up();
+    }
+  });
+
+  // Clean up before each test
+  beforeEach(() => {
+    // Clean up test data
+    db.exec("DELETE FROM document_chunks");
+    db.exec("DELETE FROM documents");
+  });
+
+  // Close the database after all tests
+  afterAll(() => {
+    db.close();
+  });
+
+  test("should index document and call generateEmbeddings with correct parameters", async () => {
+    const sampleText = "This is a test document for indexing";
+    const sampleSource = "test-source";
+
+    // Create a request with sample data
+    const request = new Request("http://localhost/index", {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        source: sampleSource,
+        text: sampleText,
+      }),
+    });
+
+    // Process the request
+    const response = await indexRoute(request);
+    const responseData = await response.json();
+
+    // Verify response
+    expect(response.status).toBe(200);
+    expect(responseData).toHaveProperty("message", "Data successfully indexed");
+
+    // Verify that generateEmbeddings was called with the correct text
+    expect(generateEmbeddingsMock.mock.calls.length).toBeGreaterThan(0);
+    const callArgs = generateEmbeddingsMock.mock.calls[0];
+    if (callArgs) {
+      expect(callArgs[0]).toBe(sampleText);
+
+      // Verify that the config was passed correctly
+      expect(callArgs[1]).toEqual({
+        apiKey: "test-key",
+        baseURL: undefined,
+      });
+    } else {
+      throw new Error("Expected generateEmbeddingsMock to be called");
+    }
+
+    // Check that the document was stored in the database
+    const documentCount = db
+      .query("SELECT COUNT(*) as count FROM documents")
+      .get() as { count: number };
+    expect(documentCount.count).toBe(1);
+
+    // Check that the document has the correct source and text
+    const document = db
+      .query("SELECT source, text FROM documents LIMIT 1")
+      .get() as { source: string; text: string };
+    expect(document.source).toBe(sampleSource);
+    expect(document.text).toBe(sampleText);
+  });
+});

tests/run.ts

@@ -0,0 +1,41 @@
+// Override Bun's test function to run each file individually
+// This ensures each test has its own database connection/environment
+import { spawnSync } from "child_process";
+
+// Exclude this file from the tests to run
+const testFiles = [
+  "./tests/splitTextIntoChunks.test.ts",
+  "./tests/indexRoute.test.ts", 
+  "./tests/searchRoute.test.ts",
+  "./tests/documentRoute.test.ts",
+  "./tests/embedding-model-change.test.ts"
+];
+
+let allTestsPassing = true;
+
+for (const testFile of testFiles) {
+  console.log(`\n--- Running tests in ${testFile} ---\n`);
+
+  // Run each test file in its own process with in-memory database
+  const result = spawnSync("bun", ["test", testFile], { 
+    stdio: "inherit",
+    env: { 
+      ...process.env, 
+      // Use in-memory database for tests
+      DATABASE_PATH: "******" 
+    }
+  });
+
+  if (result.status !== 0) {
+    allTestsPassing = false;
+    console.error(`❌ Tests in ${testFile} failed with status ${result.status}`);
+  } else {
+    console.log(`✅ Tests in ${testFile} passed`);
+  }
+}
+
+if (!allTestsPassing) {
+  process.exit(1);
+}
+
+console.log("\n✅ All tests passed!");