diff --git a/.github/workflows/cortex-cpp-quality-gate.yml b/.github/workflows/cortex-cpp-quality-gate.yml index 87be2cf47..68d4d9c09 100644 --- a/.github/workflows/cortex-cpp-quality-gate.yml +++ b/.github/workflows/cortex-cpp-quality-gate.yml @@ -168,7 +168,7 @@ jobs: cp build/cortex build/cortex-beta python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/main.py + python e2e-test/runner/main.py rm build/cortex-nightly rm build/cortex-beta env: @@ -182,7 +182,7 @@ jobs: cp build/cortex.exe build/cortex-beta.exe python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/main.py + python e2e-test/runner/main.py rm build/cortex-nightly.exe rm build/cortex-beta.exe env: @@ -196,7 +196,7 @@ jobs: cp build/cortex build/cortex-beta python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/cortex-llamacpp-e2e-nightly.py + python e2e-test/runner/cortex-llamacpp-e2e-nightly.py rm build/cortex-nightly rm build/cortex-beta env: @@ -210,7 +210,7 @@ jobs: cp build/cortex.exe build/cortex-beta.exe python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/cortex-llamacpp-e2e-nightly.py + python e2e-test/runner/cortex-llamacpp-e2e-nightly.py rm build/cortex-nightly.exe rm build/cortex-beta.exe env: @@ -443,7 +443,7 @@ jobs: cp build/cortex build/cortex-beta python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/main.py + python e2e-test/runner/main.py rm build/cortex-nightly rm build/cortex-beta env: @@ -457,7 +457,7 @@ jobs: cp build/cortex.exe build/cortex-beta.exe python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/main.py + python e2e-test/runner/main.py rm build/cortex-nightly.exe rm build/cortex-beta.exe env: @@ -471,7 +471,7 @@ jobs: cp build/cortex build/cortex-beta python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/cortex-llamacpp-e2e-nightly.py + python e2e-test/runner/cortex-llamacpp-e2e-nightly.py rm build/cortex-nightly rm build/cortex-beta env: @@ -485,7 +485,7 @@ jobs: cp build/cortex.exe build/cortex-beta.exe python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/cortex-llamacpp-e2e-nightly.py + python e2e-test/runner/cortex-llamacpp-e2e-nightly.py rm build/cortex-nightly.exe rm build/cortex-beta.exe env: diff --git a/engine/e2e-test/test_api_engine.py b/engine/e2e-test/api/engines/test_api_engine.py similarity index 99% rename from engine/e2e-test/test_api_engine.py rename to engine/e2e-test/api/engines/test_api_engine.py index e652e4495..aa491caf7 100644 --- a/engine/e2e-test/test_api_engine.py +++ b/engine/e2e-test/api/engines/test_api_engine.py @@ -1,7 +1,7 @@ import pytest import requests import time -from test_runner import ( +from utils.test_runner import ( start_server, stop_server, wait_for_websocket_download_success_event, diff --git a/engine/e2e-test/test_api_engine_install_nightly.py b/engine/e2e-test/api/engines/test_api_engine_install_nightly.py similarity index 93% rename from engine/e2e-test/test_api_engine_install_nightly.py rename to engine/e2e-test/api/engines/test_api_engine_install_nightly.py index de4914c28..34fda2d18 100644 --- a/engine/e2e-test/test_api_engine_install_nightly.py +++ b/engine/e2e-test/api/engines/test_api_engine_install_nightly.py @@ -1,6 +1,6 @@ import pytest import requests -from test_runner import start_server, stop_server, get_latest_pre_release_tag +from utils.test_runner import start_server, stop_server, get_latest_pre_release_tag latest_pre_release_tag = get_latest_pre_release_tag("janhq", "cortex.llamacpp") diff --git a/engine/e2e-test/test_api_engine_update.py b/engine/e2e-test/api/engines/test_api_engine_update.py similarity index 97% rename from engine/e2e-test/test_api_engine_update.py rename to engine/e2e-test/api/engines/test_api_engine_update.py index be8685dba..7bdc58a1f 100644 --- a/engine/e2e-test/test_api_engine_update.py +++ b/engine/e2e-test/api/engines/test_api_engine_update.py @@ -1,6 +1,6 @@ import pytest import requests -from test_runner import ( +from utils.test_runner import ( start_server, stop_server, wait_for_websocket_download_success_event, diff --git a/engine/e2e-test/api/engines/test_api_get_default_engine.py b/engine/e2e-test/api/engines/test_api_get_default_engine.py new file mode 100644 index 000000000..0320c1cc1 --- /dev/null +++ b/engine/e2e-test/api/engines/test_api_get_default_engine.py @@ -0,0 +1,82 @@ +import pytest +import requests +from utils.test_runner import start_server, stop_server +import jsonschema +from tenacity import retry, wait_exponential, stop_after_attempt +from utils.logger import log_response +from utils.assertion import assert_equal + + +class TestApiDefaultEngine: + + @pytest.fixture(autouse=True) + def setup_and_teardown(self): + # Setup + success = start_server() + if not success: + raise Exception("Failed to start server") + + yield + + # Teardown + stop_server() + + def test_api_get_default_engine_successfully(self): + # Data test + engine= "llama-cpp" + name= "linux-amd64-avx-cuda-11-7" + version= "v0.1.35-27.10.24" + + data = {"version": version, "variant": name} + post_install_url = f"http://localhost:3928/v1/engines/{engine}/install" + response = requests.post( + post_install_url, json=data + ) + assert_equal(response.status_code,200) + log_response(response.json(), "test_api_get_default_engine_successfully") + + get_list_url = f"http://localhost:3928/v1/engines/{engine}" + get_default_url = f"http://localhost:3928/v1/engines/{engine}/default" + + @retry( + wait=wait_exponential(multiplier=2, min=2, max=30), + stop=stop_after_attempt(5) + ) + def get_request(url): + response = requests.get(url) + assert len(response.json()) > 0 + + get_request(get_list_url) + + response_default_engine = requests.get(get_default_url) + json_data = response_default_engine.json() + + log_response(json_data, "test_api_get_default_engine_successfully") + assert_equal(response_default_engine.status_code, 200) + + schema = { + "type": "object", + "properties": { + "engine": {"type": "string"}, + "variant": {"type": "string"}, + "version": {"type": "string"} + }, + "required": ["engine", "variant", "version"] + } + + # Validate response schema + jsonschema.validate(instance=json_data, schema=schema) + + def test_api_get_default_engine_failed_invalid_engine(self): + # Data test + engine= "invalid" + + get_default_url = f"http://localhost:3928/v1/engines/{engine}/default" + + response_default_engine = requests.get(get_default_url) + json_data_get_default = response_default_engine.json() + + log_response(json_data_get_default, "test_api_get_default_engine_failed_invalid_engine") + assert_equal(response_default_engine.status_code, 400) + + assert_equal(json_data_get_default["message"], f"Engine {engine} is not supported yet!") \ No newline at end of file diff --git a/engine/e2e-test/api/engines/test_api_get_engine_release.py b/engine/e2e-test/api/engines/test_api_get_engine_release.py new file mode 100644 index 000000000..1e68de121 --- /dev/null +++ b/engine/e2e-test/api/engines/test_api_get_engine_release.py @@ -0,0 +1,76 @@ +import pytest +import requests +from utils.test_runner import start_server, stop_server +import jsonschema +from tenacity import retry, wait_exponential, stop_after_attempt +from utils.logger import log_response +from utils.assertion import assert_equal, assert_contains + + +class TestApiEngineRelease: + + @pytest.fixture(autouse=True) + def setup_and_teardown(self): + # Setup + success = start_server() + if not success: + raise Exception("Failed to start server") + + yield + + # Teardown + stop_server() + + def test_api_get_engine_release_successfully(self): + # Data test + engine= "llama-cpp" + get_release_url = f"http://localhost:3928/v1/engines/{engine}/releases" + + @retry( + wait=wait_exponential(multiplier=2, min=2, max=30), + stop=stop_after_attempt(5) + ) + def get_request(url): + response = requests.get(url) + assert len(response.json()) > 0 + + get_request(get_release_url) + + response_engine_release = requests.get(get_release_url) + json_data = response_engine_release.json() + + log_response(json_data, "test_api_get_engine_release_successfully") + assert_equal(response_engine_release.status_code, 200) + + schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "array", + "items": { + "type": "object", + "properties": { + "draft": { "type": "boolean" }, + "name": { "type": "string" }, + "prerelease": { "type": "boolean" }, + "published_at": { "type": "string", "format": "date-time" }, + "url": { "type": "string", "format": "uri" } + }, + "required": ["draft", "name", "prerelease", "published_at", "url"] + } + } + + # Validate response schema + jsonschema.validate(instance=json_data, schema=schema) + + def test_api_ge_engine_release_failed_invalid_engine(self): + # Data test + engine= "invalid" + + get_default_url = f"http://localhost:3928/v1/engines/{engine}/releases" + + response_default_engine = requests.get(get_default_url) + json_data_get_default = response_default_engine.json() + + log_response(json_data_get_default, "test_api_ge_engine_release_failed_invalid_engine") + assert_equal(response_default_engine.status_code, 400) + + assert_contains(json_data_get_default["message"], "Not Found") \ No newline at end of file diff --git a/engine/e2e-test/api/engines/test_api_get_engine_release_latest.py b/engine/e2e-test/api/engines/test_api_get_engine_release_latest.py new file mode 100644 index 000000000..be65141ea --- /dev/null +++ b/engine/e2e-test/api/engines/test_api_get_engine_release_latest.py @@ -0,0 +1,73 @@ +import pytest +import requests +from utils.test_runner import start_server, stop_server +import jsonschema +from tenacity import retry, wait_exponential, stop_after_attempt +from utils.logger import log_response +from utils.assertion import assert_equal, assert_contains + + +class TestApiEngineReleaseLatest: + + @pytest.fixture(autouse=True) + def setup_and_teardown(self): + # Setup + success = start_server() + if not success: + raise Exception("Failed to start server") + + yield + + # Teardown + stop_server() + + def test_api_get_engine_release_latest_successfully(self): + # Data test + engine= "llama-cpp" + get_release_url = f"http://localhost:3928/v1/engines/{engine}/releases/latest" + + @retry( + wait=wait_exponential(multiplier=2, min=2, max=30), + stop=stop_after_attempt(5) + ) + def get_request(url): + response = requests.get(url) + assert len(response.json()) > 0 + + get_request(get_release_url) + + response_engine_release = requests.get(get_release_url) + json_data = response_engine_release.json() + + log_response(json_data, "test_api_get_engine_release_latest_successfully") + assert_equal(response_engine_release.status_code, 200) + + schema = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "array", + "items": { + "type": "object", + "properties": { + "created_at": { + "type": "string", + "format": "date-time" + }, + "download_count": { + "type": "integer", + "minimum": 0 + }, + "name": { + "type": "string" + }, + "size": { + "type": "integer", + "minimum": 0 + } + }, + "required": ["created_at", "download_count", "name", "size"] + } + } + + + # Validate response schema + jsonschema.validate(instance=json_data, schema=schema) \ No newline at end of file diff --git a/engine/e2e-test/api/engines/test_api_get_list_engine.py b/engine/e2e-test/api/engines/test_api_get_list_engine.py new file mode 100644 index 000000000..9a1552de6 --- /dev/null +++ b/engine/e2e-test/api/engines/test_api_get_list_engine.py @@ -0,0 +1,73 @@ +import pytest +import requests +from utils.test_runner import start_server, stop_server +import jsonschema +from tenacity import retry, wait_exponential, stop_after_attempt +from utils.logger import log_response +from utils.assertion import assert_equal + + +class TestApiEngineList: + + @pytest.fixture(autouse=True) + def setup_and_teardown(self): + # Setup + success = start_server() + if not success: + raise Exception("Failed to start server") + + yield + + # Teardown + stop_server() + + def test_api_get_list_engines_successfully(self): + # Data test + engine= "llama-cpp" + name= "linux-amd64-avx-cuda-11-7" + version= "v0.1.35-27.10.24" + + data = {"version": version, "variant": name} + post_install_url = f"http://localhost:3928/v1/engines/{engine}/install" + response = requests.post( + post_install_url, json=data + ) + assert_equal(response.status_code,200) + + get_list_url = f"http://localhost:3928/v1/engines/{engine}" + + @retry( + wait=wait_exponential(multiplier=2, min=2, max=30), + stop=stop_after_attempt(5) + ) + def get_request(url): + response = requests.get(url) + assert len(response.json()) > 0 + return response + + response_get_list = get_request(get_list_url) + json_data = response_get_list.json() + + log_response(json_data, "test_api_get_list_engines_successfully") + assert_equal(response_get_list.status_code, 200) + + schema = { + "type": "array", + "items": { + "type": "object", + "properties": { + "engine": {"type": "string"}, + "name": {"type": "string"}, + "version": {"type": "string"} + }, + "required": ["engine", "name", "version"] + } + } + + # Validate response schema + jsonschema.validate(instance=json_data, schema=schema) + + assert_equal(len(json_data), 1) + assert_equal(json_data[0]["engine"], engine) + assert_equal(json_data[0]["version"], version) + assert_equal(json_data[0]["name"], name) \ No newline at end of file diff --git a/engine/e2e-test/test_api_model.py b/engine/e2e-test/api/model/test_api_model.py similarity index 99% rename from engine/e2e-test/test_api_model.py rename to engine/e2e-test/api/model/test_api_model.py index d75aa6831..fac94a9f1 100644 --- a/engine/e2e-test/test_api_model.py +++ b/engine/e2e-test/api/model/test_api_model.py @@ -1,7 +1,7 @@ import pytest import requests import time -from test_runner import ( +from utils.test_runner import ( run, start_server, stop_server, diff --git a/engine/e2e-test/test_api_model_import.py b/engine/e2e-test/api/model/test_api_model_import.py similarity index 97% rename from engine/e2e-test/test_api_model_import.py rename to engine/e2e-test/api/model/test_api_model_import.py index 7efbd52da..46a22e59e 100644 --- a/engine/e2e-test/test_api_model_import.py +++ b/engine/e2e-test/api/model/test_api_model_import.py @@ -1,6 +1,6 @@ import pytest import requests -from test_runner import start_server, stop_server +from utils.test_runner import start_server, stop_server class TestApiModelImport: @pytest.fixture(autouse=True) diff --git a/engine/e2e-test/test_cli_server_start.py b/engine/e2e-test/cli/common/test_cli_server_start.py similarity index 84% rename from engine/e2e-test/test_cli_server_start.py rename to engine/e2e-test/cli/common/test_cli_server_start.py index abe236cd0..549191458 100644 --- a/engine/e2e-test/test_cli_server_start.py +++ b/engine/e2e-test/cli/common/test_cli_server_start.py @@ -1,8 +1,8 @@ import platform import os import pytest, requests -from test_runner import run -from test_runner import start_server, stop_server +from utils.test_runner import run +from utils.test_runner import start_server, stop_server class TestCliServerStart: diff --git a/engine/e2e-test/test_cortex_update.py b/engine/e2e-test/cli/common/test_cortex_update.py similarity index 94% rename from engine/e2e-test/test_cortex_update.py rename to engine/e2e-test/cli/common/test_cortex_update.py index 8f6f8d7f8..7bcbdba5c 100644 --- a/engine/e2e-test/test_cortex_update.py +++ b/engine/e2e-test/cli/common/test_cortex_update.py @@ -2,7 +2,7 @@ import tempfile import pytest -from test_runner import run +from utils.test_runner import run class TestCortexUpdate: diff --git a/engine/e2e-test/test_create_log_folder.py b/engine/e2e-test/cli/common/test_create_log_folder.py similarity index 90% rename from engine/e2e-test/test_create_log_folder.py rename to engine/e2e-test/cli/common/test_create_log_folder.py index 5dbbd521c..7ad6ee452 100644 --- a/engine/e2e-test/test_create_log_folder.py +++ b/engine/e2e-test/cli/common/test_create_log_folder.py @@ -2,8 +2,8 @@ import os from pathlib import Path import pytest, requests, shutil -from test_runner import run -from test_runner import start_server, stop_server +from utils.test_runner import run +from utils.test_runner import start_server, stop_server class TestCreateLogFolder: diff --git a/engine/e2e-test/test_cli_engine_get.py b/engine/e2e-test/cli/engines/test_cli_engine_get.py similarity index 97% rename from engine/e2e-test/test_cli_engine_get.py rename to engine/e2e-test/cli/engines/test_cli_engine_get.py index 1eb32388c..4105062d6 100644 --- a/engine/e2e-test/test_cli_engine_get.py +++ b/engine/e2e-test/cli/engines/test_cli_engine_get.py @@ -1,7 +1,7 @@ import platform import pytest -from test_runner import run, start_server, stop_server +from utils.test_runner import run, start_server, stop_server class TestCliEngineGet: diff --git a/engine/e2e-test/test_cli_engine_install.py b/engine/e2e-test/cli/engines/test_cli_engine_install.py similarity index 98% rename from engine/e2e-test/test_cli_engine_install.py rename to engine/e2e-test/cli/engines/test_cli_engine_install.py index aeeabd64d..370ebe3f3 100644 --- a/engine/e2e-test/test_cli_engine_install.py +++ b/engine/e2e-test/cli/engines/test_cli_engine_install.py @@ -3,7 +3,7 @@ import pytest import requests -from test_runner import run, start_server, stop_server +from utils.test_runner import run, start_server, stop_server class TestCliEngineInstall: diff --git a/engine/e2e-test/test_cli_engine_install_nightly.py b/engine/e2e-test/cli/engines/test_cli_engine_install_nightly.py similarity index 97% rename from engine/e2e-test/test_cli_engine_install_nightly.py rename to engine/e2e-test/cli/engines/test_cli_engine_install_nightly.py index 80490ab55..376dd3c30 100644 --- a/engine/e2e-test/test_cli_engine_install_nightly.py +++ b/engine/e2e-test/cli/engines/test_cli_engine_install_nightly.py @@ -3,7 +3,7 @@ import pytest import requests -from test_runner import run, start_server, stop_server, get_latest_pre_release_tag +from utils.test_runner import run, start_server, stop_server, get_latest_pre_release_tag latest_pre_release_tag = get_latest_pre_release_tag("janhq", "cortex.llamacpp") diff --git a/engine/e2e-test/test_cli_engine_list.py b/engine/e2e-test/cli/engines/test_cli_engine_list.py similarity index 95% rename from engine/e2e-test/test_cli_engine_list.py rename to engine/e2e-test/cli/engines/test_cli_engine_list.py index 6a79bb449..b2d59e855 100644 --- a/engine/e2e-test/test_cli_engine_list.py +++ b/engine/e2e-test/cli/engines/test_cli_engine_list.py @@ -1,7 +1,7 @@ import platform import pytest -from test_runner import run, start_server, stop_server +from utils.test_runner import run, start_server, stop_server class TestCliEngineList: diff --git a/engine/e2e-test/test_cli_engine_uninstall.py b/engine/e2e-test/cli/engines/test_cli_engine_uninstall.py similarity index 96% rename from engine/e2e-test/test_cli_engine_uninstall.py rename to engine/e2e-test/cli/engines/test_cli_engine_uninstall.py index 6b640d45d..8672110e2 100644 --- a/engine/e2e-test/test_cli_engine_uninstall.py +++ b/engine/e2e-test/cli/engines/test_cli_engine_uninstall.py @@ -1,6 +1,6 @@ import pytest import requests -from test_runner import ( +from utils.test_runner import ( run, start_server, stop_server, diff --git a/engine/e2e-test/test_cli_model.py b/engine/e2e-test/cli/model/test_cli_model.py similarity index 98% rename from engine/e2e-test/test_cli_model.py rename to engine/e2e-test/cli/model/test_cli_model.py index f6aad4ae9..63261c214 100644 --- a/engine/e2e-test/test_cli_model.py +++ b/engine/e2e-test/cli/model/test_cli_model.py @@ -2,7 +2,7 @@ import requests import os from pathlib import Path -from test_runner import ( +from utils.test_runner import ( run, start_server, stop_server, diff --git a/engine/e2e-test/test_cli_model_import.py b/engine/e2e-test/cli/model/test_cli_model_import.py similarity index 89% rename from engine/e2e-test/test_cli_model_import.py rename to engine/e2e-test/cli/model/test_cli_model_import.py index cf94d1a2a..e2c5010a4 100644 --- a/engine/e2e-test/test_cli_model_import.py +++ b/engine/e2e-test/cli/model/test_cli_model_import.py @@ -1,6 +1,6 @@ import pytest -from test_runner import run -from test_runner import start_server, stop_server +from utils.test_runner import run +from utils.test_runner import start_server, stop_server class TestCliModelImport: diff --git a/engine/e2e-test/test_cli_model_pull_cortexso_with_selection.py b/engine/e2e-test/cli/model/test_cli_model_pull_cortexso_with_selection.py similarity index 94% rename from engine/e2e-test/test_cli_model_pull_cortexso_with_selection.py rename to engine/e2e-test/cli/model/test_cli_model_pull_cortexso_with_selection.py index 8c3de8d98..d6a78843b 100644 --- a/engine/e2e-test/test_cli_model_pull_cortexso_with_selection.py +++ b/engine/e2e-test/cli/model/test_cli_model_pull_cortexso_with_selection.py @@ -1,4 +1,4 @@ -from test_runner import popen +from utils.test_runner import popen import os from pathlib import Path diff --git a/engine/e2e-test/test_cli_model_pull_from_cortexso.py b/engine/e2e-test/cli/model/test_cli_model_pull_from_cortexso.py similarity index 95% rename from engine/e2e-test/test_cli_model_pull_from_cortexso.py rename to engine/e2e-test/cli/model/test_cli_model_pull_from_cortexso.py index 1791e39a6..2a66ae19e 100644 --- a/engine/e2e-test/test_cli_model_pull_from_cortexso.py +++ b/engine/e2e-test/cli/model/test_cli_model_pull_from_cortexso.py @@ -1,5 +1,5 @@ import pytest -from test_runner import run +from utils.test_runner import run class TestCliModelPullCortexso: diff --git a/engine/e2e-test/test_cli_model_pull_hugging_face_repository.py b/engine/e2e-test/cli/model/test_cli_model_pull_hugging_face_repository.py similarity index 96% rename from engine/e2e-test/test_cli_model_pull_hugging_face_repository.py rename to engine/e2e-test/cli/model/test_cli_model_pull_hugging_face_repository.py index 996ac086c..49678ee7a 100644 --- a/engine/e2e-test/test_cli_model_pull_hugging_face_repository.py +++ b/engine/e2e-test/cli/model/test_cli_model_pull_hugging_face_repository.py @@ -1,5 +1,5 @@ import pytest -from test_runner import popen +from utils.test_runner import popen class TestCliModelPullHuggingFaceRepository: diff --git a/engine/e2e-test/cortex-llamacpp-e2e-nightly.py b/engine/e2e-test/cortex-llamacpp-e2e-nightly.py deleted file mode 100644 index 0511277f3..000000000 --- a/engine/e2e-test/cortex-llamacpp-e2e-nightly.py +++ /dev/null @@ -1,22 +0,0 @@ -import pytest -import sys - -### e2e tests are expensive, have to keep engines tests in order -from test_api_engine_list import TestApiEngineList -from test_api_engine_install_nightly import TestApiEngineInstall -from test_api_model import TestApiModel -from test_api_model_import import TestApiModelImport - -### -from test_cli_engine_get import TestCliEngineGet -from test_cli_engine_install_nightly import TestCliEngineInstall -from test_cli_engine_list import TestCliEngineList -from test_cli_engine_uninstall import TestCliEngineUninstall -from test_cli_model import TestCliModel -from test_cli_server_start import TestCliServerStart -from test_cortex_update import TestCortexUpdate -from test_create_log_folder import TestCreateLogFolder -from test_cli_model_import import TestCliModelImport - -if __name__ == "__main__": - sys.exit(pytest.main([__file__, "-v"])) diff --git a/engine/e2e-test/main.py b/engine/e2e-test/main.py deleted file mode 100644 index e874ab3a0..000000000 --- a/engine/e2e-test/main.py +++ /dev/null @@ -1,22 +0,0 @@ -import pytest -import sys - -### e2e tests are expensive, have to keep engines tests in order -from test_api_engine_list import TestApiEngineList -from test_api_engine import TestApiEngine -from test_api_model import TestApiModel -from test_api_model_import import TestApiModelImport - -### -from test_cli_engine_get import TestCliEngineGet -from test_cli_engine_install import TestCliEngineInstall -from test_cli_engine_list import TestCliEngineList -from test_cli_engine_uninstall import TestCliEngineUninstall -from test_cli_model import TestCliModel -from test_cli_server_start import TestCliServerStart -from test_cortex_update import TestCortexUpdate -from test_create_log_folder import TestCreateLogFolder -from test_cli_model_import import TestCliModelImport - -if __name__ == "__main__": - sys.exit(pytest.main([__file__, "-v"])) diff --git a/engine/e2e-test/requirements.txt b/engine/e2e-test/requirements.txt index 05b47e0b0..6991b03d3 100644 --- a/engine/e2e-test/requirements.txt +++ b/engine/e2e-test/requirements.txt @@ -2,4 +2,6 @@ websockets pytest pytest-asyncio requests -pyyaml \ No newline at end of file +pyyaml +jsonschema +tenacity \ No newline at end of file diff --git a/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py b/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py new file mode 100644 index 000000000..61b153267 --- /dev/null +++ b/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py @@ -0,0 +1,39 @@ +import pytest +import sys +import os + +# Add the project root to sys.path +PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + + +# Add all necessary paths +sys.path.append(PROJECT_ROOT) +sys.path.append(os.path.join(PROJECT_ROOT, "api/engines")) +sys.path.append(os.path.join(PROJECT_ROOT, "api/hub")) +sys.path.append(os.path.join(PROJECT_ROOT, "api/model")) +sys.path.append(os.path.join(PROJECT_ROOT, "cli/engines")) +sys.path.append(os.path.join(PROJECT_ROOT, "cli/model")) +sys.path.append(os.path.join(PROJECT_ROOT, "cli/common")) + +### e2e tests are expensive, have to keep engines tests in order +from api.engines.test_api_get_list_engine import TestApiEngineList +from api.engines.test_api_engine_install_nightly import TestApiEngineInstall +from api.engines.test_api_get_default_engine import TestApiDefaultEngine +from api.engines.test_api_get_engine_release import TestApiEngineRelease +from api.engines.test_api_get_engine_release_latest import TestApiEngineReleaseLatest +from api.model.test_api_model import TestApiModel +from api.model.test_api_model_import import TestApiModelImport + +### +from cli.engines.test_cli_engine_get import TestCliEngineGet +from cli.engines.test_cli_engine_install_nightly import TestCliEngineInstall +from cli.engines.test_cli_engine_list import TestCliEngineList +from cli.engines.test_cli_engine_uninstall import TestCliEngineUninstall +from cli.model.test_cli_model import TestCliModel +from cli.model.test_cli_model_import import TestCliModelImport +from cli.common.test_cli_server_start import TestCliServerStart +from cli.common.test_cortex_update import TestCortexUpdate +from cli.common.test_create_log_folder import TestCreateLogFolder + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/engine/e2e-test/runner/main.py b/engine/e2e-test/runner/main.py new file mode 100644 index 000000000..9b2a0316c --- /dev/null +++ b/engine/e2e-test/runner/main.py @@ -0,0 +1,39 @@ +import pytest +import sys +import os + +# Add the project root to sys.path +PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + + +# Add all necessary paths +sys.path.append(PROJECT_ROOT) +sys.path.append(os.path.join(PROJECT_ROOT, "api/engines")) +sys.path.append(os.path.join(PROJECT_ROOT, "api/hub")) +sys.path.append(os.path.join(PROJECT_ROOT, "api/model")) +sys.path.append(os.path.join(PROJECT_ROOT, "cli/engines")) +sys.path.append(os.path.join(PROJECT_ROOT, "cli/model")) +sys.path.append(os.path.join(PROJECT_ROOT, "cli/common")) + +### e2e tests are expensive, have to keep engines tests in order +from api.engines.test_api_get_list_engine import TestApiEngineList +from api.engines.test_api_engine import TestApiEngine +from api.engines.test_api_get_default_engine import TestApiDefaultEngine +from api.engines.test_api_get_engine_release import TestApiEngineRelease +from api.engines.test_api_get_engine_release_latest import TestApiEngineReleaseLatest +from api.model.test_api_model import TestApiModel +from api.model.test_api_model_import import TestApiModelImport + +### +from cli.engines.test_cli_engine_get import TestCliEngineGet +from cli.engines.test_cli_engine_install import TestCliEngineInstall +from cli.engines.test_cli_engine_list import TestCliEngineList +from cli.engines.test_cli_engine_uninstall import TestCliEngineUninstall +from cli.model.test_cli_model import TestCliModel +from cli.model.test_cli_model_import import TestCliModelImport +from cli.common.test_cli_server_start import TestCliServerStart +from cli.common.test_cortex_update import TestCortexUpdate +from cli.common.test_create_log_folder import TestCreateLogFolder + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/engine/e2e-test/test_api_cortexso_hub_llamacpp_engine.py b/engine/e2e-test/test_api_cortexso_hub_llamacpp_engine.py index 9aecd3654..7a3c2e232 100644 --- a/engine/e2e-test/test_api_cortexso_hub_llamacpp_engine.py +++ b/engine/e2e-test/test_api_cortexso_hub_llamacpp_engine.py @@ -4,7 +4,7 @@ import yaml from pathlib import Path -from test_runner import ( +from utils.test_runner import ( run, start_server, stop_server, diff --git a/engine/e2e-test/test_api_docker.py b/engine/e2e-test/test_api_docker.py index b46b1f782..7276f2078 100644 --- a/engine/e2e-test/test_api_docker.py +++ b/engine/e2e-test/test_api_docker.py @@ -1,6 +1,6 @@ import pytest import requests -from test_runner import wait_for_websocket_download_success_event +from utils.test_runner import wait_for_websocket_download_success_event repo_branches = ["tinyllama:1b-gguf"] diff --git a/engine/e2e-test/test_api_engine_list.py b/engine/e2e-test/test_api_engine_list.py deleted file mode 100644 index f149f1450..000000000 --- a/engine/e2e-test/test_api_engine_list.py +++ /dev/null @@ -1,25 +0,0 @@ -import pytest -import requests -from test_runner import start_server, stop_server - - -class TestApiEngineList: - - @pytest.fixture(autouse=True) - def setup_and_teardown(self): - # Setup - # Not sure why but on macOS amd, the first start server timeouts with CI - start_server() - stop_server() - success = start_server() - if not success: - raise Exception("Failed to start server") - - yield - - # Teardown - stop_server() - - def test_engines_list_api_run_successfully(self): - response = requests.get("http://localhost:3928/engines") - assert response.status_code == 200 \ No newline at end of file diff --git a/engine/e2e-test/utils/assertion.py b/engine/e2e-test/utils/assertion.py new file mode 100644 index 000000000..63443981f --- /dev/null +++ b/engine/e2e-test/utils/assertion.py @@ -0,0 +1,7 @@ +def assert_equal(actual, expected): + """Custom assertion to compare actual and expected values.""" + assert actual == expected, f"Assertion failed: Expected '{expected}', but got '{actual}'" + +def assert_contains(main_string, sub_string): + """Custom assertion to compare actual and expected values.""" + assert sub_string in main_string, f"Assertion failed: Expected '{main_string}' has '{sub_string}'" diff --git a/engine/e2e-test/utils/logger.py b/engine/e2e-test/utils/logger.py new file mode 100644 index 000000000..578a4f2cc --- /dev/null +++ b/engine/e2e-test/utils/logger.py @@ -0,0 +1,15 @@ +import json +import os + +def log_response(data, test_name): + """Log the data to a file named after the test.""" + log_dir="e2e-test/logs" + os.makedirs(log_dir, exist_ok=True) # Ensure log directory exists + file_path = os.path.join(log_dir, f"{test_name}.txt") # Log file per test + + try: + with open(file_path, "a", encoding="utf-8") as file: + json.dump(data, file, indent=4) + file.write("\n") # Ensure a new line between entries + except Exception as e: + print(f"Error logging response: {e}") diff --git a/engine/e2e-test/test_runner.py b/engine/e2e-test/utils/test_runner.py similarity index 100% rename from engine/e2e-test/test_runner.py rename to engine/e2e-test/utils/test_runner.py