Skip to content

Commit

Permalink
Implement the first real test of gpt4all-chat (#3116)
Browse files Browse the repository at this point in the history
Signed-off-by: Jared Van Bortel <jared@nomic.ai>
  • Loading branch information
cebtenzzre authored Oct 20, 2024
1 parent 9cafd38 commit 7f5f086
Show file tree
Hide file tree
Showing 14 changed files with 157 additions and 13 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,7 @@ gpt4all-chat/models/*
build_*
build-*
cmake-build-*
/gpt4all-chat/tests/python/config.py

# IntelliJ
.idea/
Expand Down
5 changes: 5 additions & 0 deletions gpt4all-chat/.flake8
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# vim: set syntax=dosini:
[flake8]
exclude = .*,__pycache__
max-line-length = 120
extend-ignore = B001,C408,D,DAR,E221,E303,E722,E741,E800,N801,N806,P101,S101,S324,S404,S406,S410,S603,WPS100,WPS110,WPS111,WPS113,WPS114,WPS115,WPS120,WPS2,WPS300,WPS301,WPS304,WPS305,WPS306,WPS309,WPS316,WPS317,WPS318,WPS319,WPS322,WPS323,WPS326,WPS329,WPS330,WPS332,WPS336,WPS337,WPS347,WPS360,WPS361,WPS414,WPS420,WPS421,WPS429,WPS430,WPS431,WPS432,WPS433,WPS437,WPS440,WPS440,WPS441,WPS442,WPS457,WPS458,WPS460,WPS462,WPS463,WPS473,WPS501,WPS504,WPS505,WPS508,WPS509,WPS510,WPS515,WPS516,WPS519,WPS529,WPS531,WPS602,WPS604,WPS605,WPS608,WPS609,WPS613,WPS615
4 changes: 2 additions & 2 deletions gpt4all-chat/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ if(APPLE)
endif()
endif()

find_package(Python3 QUIET COMPONENTS Interpreter)
find_package(Python3 3.12 QUIET COMPONENTS Interpreter)

option(GPT4ALL_TEST "Build the tests" ${Python3_FOUND})
option(GPT4ALL_LOCALHOST "Build installer for localhost repo" OFF)
Expand Down Expand Up @@ -101,7 +101,7 @@ if (GPT4ALL_TEST)
add_subdirectory(tests)

# The 'check' target makes sure the tests and their dependencies are up-to-date before running them
add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} DEPENDS chat gpt4all_tests)
add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure DEPENDS chat gpt4all_tests)
endif()

set(CHAT_EXE_RESOURCES)
Expand Down
11 changes: 11 additions & 0 deletions gpt4all-chat/dev-requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
-r test-requirements.txt

# dev tools
flake8~=7.1
mypy~=1.12
pytype>=2024.10.11
wemake-python-styleguide~=0.19.2

# type stubs and other optional modules
types-requests~=2.32
urllib3[socks]
29 changes: 29 additions & 0 deletions gpt4all-chat/pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
[tool.pytest.ini_options]
addopts = ['--import-mode=importlib']

[tool.mypy]
files = 'tests/python'
pretty = true
strict = true
warn_unused_ignores = false

[tool.pytype]
inputs = ['tests/python']
jobs = 'auto'
bind_decorated_methods = true
none_is_not_bool = true
overriding_renamed_parameter_count_checks = true
strict_none_binding = true
precise_return = true
# protocols:
# - https://github.com/google/pytype/issues/1423
# - https://github.com/google/pytype/issues/1424
strict_import = true
strict_parameter_checks = true
strict_primitive_comparisons = true
# strict_undefined_checks: too many false positives

[tool.isort]
src_paths = ['tests/python']
line_length = 120
combine_as_imports = true
13 changes: 13 additions & 0 deletions gpt4all-chat/src/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@

#ifdef Q_OS_WINDOWS
# include <windows.h>
#else
# include <signal.h>
#endif

using namespace Qt::Literals::StringLiterals;
Expand Down Expand Up @@ -130,6 +132,17 @@ int main(int argc, char *argv[])
}
#endif

#ifndef Q_OS_WINDOWS
// handle signals gracefully
struct sigaction sa;
sa.sa_handler = [](int s) { QCoreApplication::exit(s == SIGINT ? 0 : 1); };
sa.sa_flags = SA_RESETHAND;
sigemptyset(&sa.sa_mask);
sigaction(SIGINT, &sa, nullptr);
sigaction(SIGTERM, &sa, nullptr);
sigaction(SIGHUP, &sa, nullptr);
#endif

int res = app.exec();

// Make sure ChatLLM threads are joined before global destructors run.
Expand Down
2 changes: 2 additions & 0 deletions gpt4all-chat/test-requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
pytest~=8.3
requests~=2.32
10 changes: 6 additions & 4 deletions gpt4all-chat/tests/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
include(FetchContent)

find_package(Python3 REQUIRED COMPONENTS Interpreter)
find_package(Python3 3.12 REQUIRED COMPONENTS Interpreter)

# Google test download and setup
FetchContent_Declare(
Expand All @@ -9,17 +9,19 @@ FetchContent_Declare(
)
FetchContent_MakeAvailable(googletest)

configure_file(python/config.py.in "${CMAKE_CURRENT_SOURCE_DIR}/python/config.py")

add_test(NAME ChatPythonTests
COMMAND ${Python3_EXECUTABLE} -m pytest ${CMAKE_SOURCE_DIR}/tests/python_tests
COMMAND ${Python3_EXECUTABLE} -m pytest --color=yes "${CMAKE_CURRENT_SOURCE_DIR}/python"
)
set_tests_properties(ChatPythonTests PROPERTIES
ENVIRONMENT "CHAT_EXECUTABLE=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/chat"
TIMEOUT 60
)

add_executable(gpt4all_tests
test_main.cpp
basic_test.cpp
cpp/test_main.cpp
cpp/basic_test.cpp
)

target_link_libraries(gpt4all_tests PRIVATE gtest gtest_main)
Expand Down
File renamed without changes.
File renamed without changes.
Empty file.
1 change: 1 addition & 0 deletions gpt4all-chat/tests/python/config.py.in
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
APP_VERSION = '@APP_VERSION@'
87 changes: 87 additions & 0 deletions gpt4all-chat/tests/python/test_server_api.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
import os
import signal
import subprocess
import sys
import tempfile
import textwrap
from pathlib import Path
from subprocess import CalledProcessError
from typing import Any, Iterator

import pytest
import requests
from urllib3 import Retry

from . import config


class Requestor:

Check failure on line 18 in gpt4all-chat/tests/python/test_server_api.py

View workflow job for this annotation

GitHub Actions / Check for spelling errors

Requestor ==> Requester
def __init__(self) -> None:
self.session = requests.Session()
self.http_adapter = self.session.adapters['http://']

def get(self, path: str, *, wait: bool = False) -> Any:
return self._request('GET', path, wait)

def _request(self, method: str, path: str, wait: bool) -> Any:
if wait:
retry = Retry(total=None, connect=10, read=False, status=0, other=0, backoff_factor=.01)
else:
retry = Retry(total=False)
self.http_adapter.max_retries = retry # type: ignore[attr-defined]

resp = self.session.request(method, f'http://localhost:4891/v1/{path}')
resp.raise_for_status()
return resp.json()


request = Requestor()

Check failure on line 38 in gpt4all-chat/tests/python/test_server_api.py

View workflow job for this annotation

GitHub Actions / Check for spelling errors

Requestor ==> Requester


@pytest.fixture
def chat_server_config() -> Iterator[dict[str, str]]:
if os.name != 'posix' or sys.platform == 'darwin':
pytest.skip('Need non-Apple Unix to use alternate config path')

with tempfile.TemporaryDirectory(prefix='gpt4all-test') as td:
tmpdir = Path(td)
xdg_confdir = tmpdir / 'config'
app_confdir = xdg_confdir / 'nomic.ai'
app_confdir.mkdir(parents=True)
with open(app_confdir / 'GPT4All.ini', 'w') as conf:
conf.write(textwrap.dedent(f"""\
[General]
serverChat=true
[download]
lastVersionStarted={config.APP_VERSION}
[network]
isActive=false
usageStatsActive=false
"""))
yield dict(
os.environ,
XDG_CACHE_HOME=str(tmpdir / 'cache'),
XDG_DATA_HOME=str(tmpdir / 'share'),
XDG_CONFIG_HOME=str(xdg_confdir),
APPIMAGE=str(tmpdir), # hack to bypass SingleApplication
)


@pytest.fixture
def chat_server(chat_server_config: dict[str, str]) -> Iterator[None]:
chat_executable = Path(os.environ['CHAT_EXECUTABLE']).absolute()
with subprocess.Popen(chat_executable, env=chat_server_config) as process:
try:
yield
except:
process.kill()
raise
process.send_signal(signal.SIGINT)
if retcode := process.wait():
raise CalledProcessError(retcode, process.args)


def test_list_models_empty(chat_server: None) -> None:
assert request.get('models', wait=True) == {'object': 'list', 'data': []}
7 changes: 0 additions & 7 deletions gpt4all-chat/tests/python_tests/test_executable.py

This file was deleted.

0 comments on commit 7f5f086

Please sign in to comment.