test: Isolated testing and SentryOptions tests

.github/workflows/unit_tests.yml

@@ -52,7 +52,7 @@ jobs:
           chmod u+x ${bin}
           ls -l
           ./${bin} --version
-          echo "GODOT_BIN=${GITHUB_WORKSPACE}/godot/${bin}" >> $GITHUB_ENV
+          echo "GODOT=${GITHUB_WORKSPACE}/godot/${bin}" >> $GITHUB_ENV
 
       - name: Diagnostic info
         shell: bash
@@ -66,15 +66,22 @@ jobs:
           scons project/addons/gdUnit4
           chmod +x project/addons/sentrysdk/bin/{linux,macos}/crashpad_handler
           echo "--- Rebuilding import cache.."
-          ${GODOT_BIN} --headless --editor --path project/ --quit-after 2000 || true
+          ${GODOT} --headless --editor --path project/ --quit-after 2000 || true
           echo "--- Finished rebuilding import cache."
 
       - name: Run tests
         shell: bash
         timeout-minutes: 5
         run: |
           # Exit status codes: 0 - success, 100 - ends with test failures, 101 - ends with test warnings.
-          ${GODOT_BIN} --headless --path project/ -s -d "res://addons/gdUnit4/bin/GdUnitCmdTool.gd" --ignoreHeadlessMode -c -a test
+          ${GODOT} --headless --path project/ -s -d "res://addons/gdUnit4/bin/GdUnitCmdTool.gd" --ignoreHeadlessMode -c -a test/suites/
+
+      - name: Run isolated tests
+        if: success() || failure()
+        shell: bash
+        timeout-minutes: 5
+        run: |
+          ./scripts/run-isolated-tests.sh
 
       - name: Upload results
         if: always() # do this step even if the tests fail

.gitignore

@@ -2,6 +2,7 @@
 project/addons
 project/export_presets.cfg
 project/.vscode
+project/reports
 
 src/sdk_version.gen.h
 

project/example_configuration.gd

@@ -16,6 +16,9 @@ func _configure(options: SentryOptions) -> void:
 	options.before_send = _before_send
 	options.on_crash = _on_crash
 
+	# Unit testing hooks (if you're exploring the demo project, pretend the following line doesn't exist).
+	TestingConfiguration.configure_options(options)
+
 
 ## before_send callback example
 func _before_send(ev: SentryEvent) -> SentryEvent:

project/test/isolated/configuration.gd

@@ -0,0 +1,20 @@
+class_name TestingConfiguration
+extends RefCounted
+
+
+## Detects whether an isolated test suite is running and calls its static `configure_options()` method.
+## - Since SentryOptions can be configured only once, such test suites must be executed separately.
+## - This method is called from "example_configuration.gd".
+static func configure_options(options: SentryOptions):
+	var args: PackedStringArray = OS.get_cmdline_args()
+	var idx := args.find("-a")
+	if idx == -1 or args.size() == idx + 1:
+		return
+	var path := "res://" + args[idx + 1]
+	if not path.ends_with(".gd"):
+		return
+	if not FileAccess.file_exists(path):
+		return
+	var scr: GDScript = load(path)
+	if scr.has_method(&"configure_options"):
+		scr.call(&"configure_options", options)

project/test/isolated/test_limit_events_per_frame.gd

@@ -0,0 +1,35 @@
+extends GdUnitTestSuite
+## Test "events_per_frame" error logger limit.
+
+
+signal callback_processed
+
+var _num_events: int = 0
+
+
+static func configure_options(options: SentryOptions) -> void:
+	# Only one error is allowed to be logged as event per processed frame.
+	options.error_logger_limits.events_per_frame = 1
+	# Make sure other limits are not interfering.
+	options.error_logger_limits.repeated_error_window_ms = 0
+	options.error_logger_limits.throttle_events = 88
+
+
+func before_test() -> void:
+	SentrySDK._set_before_send(_before_send)
+
+
+func _before_send(_ev: SentryEvent) -> SentryEvent:
+	_num_events += 1
+	callback_processed.emit()
+	return null
+
+
+## Only one error should be logged within 1 processed frame.
+func test_events_per_frame_limit() -> void:
+	push_error("dummy-error")
+	push_error("dummy-error")
+	push_error("dummy-error")
+	assert_signal(self).is_emitted("callback_processed")
+	await get_tree().create_timer(0.1).timeout
+	assert_int(_num_events).is_equal(1)

project/test/isolated/test_limit_repeating_error_window.gd

@@ -0,0 +1,41 @@
+extends GdUnitTestSuite
+## Test "repeated_error_window_ms" error logger limit.
+
+
+signal callback_processed
+
+var _num_events: int = 0
+
+
+static func configure_options(options: SentryOptions) -> void:
+	# Ignore duplicate errors within 1 second window.
+	options.error_logger_limits.repeated_error_window_ms = 1000
+	# Make sure other limits are not interfering.
+	options.error_logger_limits.events_per_frame = 88
+	options.error_logger_limits.throttle_events = 88
+
+
+func before_test() -> void:
+	SentrySDK._set_before_send(_before_send)
+
+
+func _before_send(_ev: SentryEvent) -> SentryEvent:
+	_num_events += 1
+	callback_processed.emit()
+	return null
+
+
+## Only one error should be logged within 1 second time window, and another one after 1 second passes.
+func test_repeating_error_window_limit() -> void:
+	push_error("dummy-error")
+	push_error("dummy-error")
+	await assert_signal(self).is_emitted("callback_processed")
+	assert_int(_num_events).is_equal(1)
+
+	# Wait for 1 second window to expire.
+	await get_tree().create_timer(1.0).timeout
+
+	push_error("dummy-error")
+	push_error("dummy-error")
+	await assert_signal(self).is_emitted("callback_processed")
+	assert_int(_num_events).is_equal(2)

project/test/isolated/test_limit_throttling.gd

@@ -0,0 +1,48 @@
+extends GdUnitTestSuite
+## Test error logger throttling limits.
+
+
+signal callback_processed
+
+var _num_events: int = 0
+
+
+static func configure_options(options: SentryOptions) -> void:
+	# Allow only two errors to be logged as events within 1 second time window.
+	options.error_logger_limits.throttle_events = 2
+	options.error_logger_limits.throttle_window_ms = 1000
+	# Make sure other limits are not interfering.
+	options.error_logger_limits.events_per_frame = 88
+	options.error_logger_limits.repeated_error_window_ms = 0
+
+
+func before_test() -> void:
+	SentrySDK._set_before_send(_before_send)
+
+
+func _before_send(_ev: SentryEvent) -> SentryEvent:
+	_num_events += 1
+	callback_processed.emit()
+	return null
+
+
+## Only two errors should be logged within the assigned time window.
+func test_throttling_limits() -> void:
+	push_error("dummy-error")
+	push_error("dummy-error")
+	push_error("dummy-error")
+	await assert_signal(self).is_emitted("callback_processed")
+	await assert_signal(self).is_emitted("callback_processed")
+	await get_tree().create_timer(0.1).timeout
+	assert_int(_num_events).is_equal(2)
+
+	# Wait for throttling window to expire.
+	await get_tree().create_timer(1.0).timeout
+
+	push_error("dummy-error")
+	push_error("dummy-error")
+	push_error("dummy-error")
+	await assert_signal(self).is_emitted("callback_processed")
+	await assert_signal(self).is_emitted("callback_processed")
+	await get_tree().create_timer(0.1).timeout
+	assert_int(_num_events).is_equal(4)

project/test/isolated/test_logger_disabled.gd

@@ -0,0 +1,36 @@
+extends GdUnitTestSuite
+## Events should not be logged for errors when the logger is disabled.
+
+
+signal callback_processed
+
+var _num_events: int = 0
+
+
+static func configure_options(options: SentryOptions) -> void:
+    options.error_logger_enabled = false
+
+    # Make sure other limits are not interfering.
+    options.error_logger_limits.events_per_frame = 88
+    options.error_logger_limits.throttle_events = 88
+    options.error_logger_limits.repeated_error_window_ms = 0
+    options.error_logger_limits.throttle_window_ms = 0
+
+
+func before_test() -> void:
+    SentrySDK._set_before_send(_before_send)
+
+
+func _before_send(_ev: SentryEvent) -> SentryEvent:
+    _num_events += 1
+    callback_processed.emit()
+    return null
+
+
+func test_event_and_breadcrumb_masks() -> void:
+    push_error("dummy-error")
+    push_warning("dummy-warning")
+
+    await assert_signal(self).is_not_emitted("callback_processed")
+
+    assert_int(_num_events).is_equal(0)

project/test/isolated/test_logger_with_masks.gd

@@ -0,0 +1,41 @@
+extends GdUnitTestSuite
+## Events and breadcrumbs should be logged when "error_logger_event_mask" and
+## "error_logger_breadcrumb_mask" are configured to include all categories.
+
+
+signal callback_processed
+
+var _num_events: int = 0
+
+
+static func configure_options(options: SentryOptions) -> void:
+    var mask = SentryOptions.MASK_ERROR | SentryOptions.MASK_SCRIPT | SentryOptions.MASK_SHADER | SentryOptions.MASK_WARNING
+    options.error_logger_event_mask = mask
+    options.error_logger_breadcrumb_mask = mask
+
+    # Make sure other limits are not interfering.
+    options.error_logger_limits.events_per_frame = 88
+    options.error_logger_limits.throttle_events = 88
+    options.error_logger_limits.repeated_error_window_ms = 0
+    options.error_logger_limits.throttle_window_ms = 0
+
+
+func before_test() -> void:
+    SentrySDK._set_before_send(_before_send)
+
+
+func _before_send(_ev: SentryEvent) -> SentryEvent:
+    _num_events += 1
+    callback_processed.emit()
+    return null
+
+
+## Both events or breadcrumbs should be logged for error and warning.
+## TODO: can't verify breadcrumbs yet, maybe later.
+func test_event_and_breadcrumb_masks() -> void:
+    push_error("dummy-error")
+    push_warning("dummy-warning")
+    await assert_signal(self).is_emitted("callback_processed")
+
+    await get_tree().create_timer(0.1).timeout
+    assert_int(_num_events).is_equal(2)

project/test/isolated/test_logger_with_masks_empty.gd

@@ -0,0 +1,34 @@
+extends GdUnitTestSuite
+## Events and breadcrumbs should not be logged when both "error_logger_event_mask"
+## and "error_logger_breadcrumb_mask" are set to zero.
+
+
+signal callback_processed
+
+var _num_events: int = 0
+
+
+static func configure_options(options: SentryOptions) -> void:
+    options.error_logger_event_mask = 0
+    options.error_logger_breadcrumb_mask = 0
+
+
+func before_test() -> void:
+    SentrySDK._set_before_send(_before_send)
+
+
+func _before_send(_ev: SentryEvent) -> SentryEvent:
+    _num_events += 1
+    callback_processed.emit()
+    return null
+
+
+## No events or breadcrumbs should be logged for errors.
+## TODO: can't verify breadcrumbs yet, maybe later.
+func test_event_and_breadcrumb_masks() -> void:
+    push_error("dummy-error")
+    push_warning("dummy-warning")
+    await assert_signal(self).is_not_emitted("callback_processed")
+
+    await get_tree().create_timer(0.1).timeout
+    assert_int(_num_events).is_equal(0)

project/test/isolated/test_options_integrity.gd

@@ -0,0 +1,28 @@
+extends GdUnitTestSuite
+## Verify that the options set in a configuration callback are correctly reflected in event objects.
+
+
+signal callback_processed
+
+
+static func configure_options(options: SentryOptions) -> void:
+	options.release = "1.2.3"
+	options.environment = "testing"
+
+
+func before_test() -> void:
+	SentrySDK._set_before_send(_before_send)
+
+
+func _before_send(ev: SentryEvent) -> SentryEvent:
+	assert_str(ev.release).is_equal("1.2.3")
+	assert_str(ev.environment).is_equal("testing")
+	callback_processed.emit()
+	return null
+
+
+## Verify that the options are correctly propagated to event objects.
+func test_options_integrity() -> void:
+	var ev := SentrySDK.create_event()
+	SentrySDK.capture_event(ev)
+	assert_signal(self).is_emitted("callback_processed")

project/test/test_event.gd → project/test/suites/test_event.gd

@@ -1,5 +1,5 @@
-class_name TestEvent
 extends GdUnitTestSuite
+## Basic tests for the SentryEvent class.
 
 
 ## SentryEvent.id should not be empty on event creation.

project/test/test_event_integrity.gd → project/test/suites/test_event_integrity.gd

@@ -1,5 +1,6 @@
-class_name TestEventIntegrity
 extends GdUnitTestSuite
+## Verify that event properties are preserved through the SDK flow.
+
 
 signal callback_processed