migrate base generator and base generator tests

garak/generators/base.py

@@ -10,6 +10,7 @@
 import tqdm
 
 from garak import _config
+from garak.attempt import Turn
 from garak.configurable import Configurable
 import garak.resources.theme
 
@@ -58,8 +59,8 @@ def __init__(self, name="", config_root=_config):
         logging.info("generator init: %s", self)
 
     def _call_model(
-        self, prompt: str, generations_this_call: int = 1
-    ) -> List[Union[str, None]]:
+        self, prompt: Turn, generations_this_call: int = 1
+    ) -> List[Union[Turn, None]]:
         """Takes a prompt and returns an API output
 
         _call_api() is fully responsible for the request, and should either
@@ -73,21 +74,21 @@ def _pre_generate_hook(self):
         pass
 
     @staticmethod
-    def _verify_model_result(result: List[Union[str, None]]):
+    def _verify_model_result(result: List[Union[Turn, None]]):
         assert isinstance(result, list), "_call_model must return a list"
         assert (
             len(result) == 1
         ), f"_call_model must return a list of one item when invoked as _call_model(prompt, 1), got {result}"
         assert (
-            isinstance(result[0], str) or result[0] is None
-        ), "_call_model's item must be a string or None"
+            isinstance(result[0], Turn) or result[0] is None
+        ), "_call_model's item must be a Turn or None"
 
     def clear_history(self):
         pass
 
     def generate(
-        self, prompt: str, generations_this_call: int = 1
-    ) -> List[Union[str, None]]:
+        self, prompt: Turn, generations_this_call: int = 1
+    ) -> List[Union[Turn, None]]:
         """Manages the process of getting generations out from a prompt
 
         This will involve iterating through prompts, getting the generations

tests/generators/test_generators.py

@@ -8,6 +8,7 @@
 
 from garak import _plugins
 from garak import _config
+from garak.attempt import Turn
 from garak.generators.test import Blank, Repeat, Single
 from garak.generators.base import Generator
 
@@ -22,51 +23,51 @@
 
 def test_generators_test_blank():
     g = Blank(DEFAULT_GENERATOR_NAME)
-    output = g.generate(prompt="test", generations_this_call=5)
+    output = g.generate(prompt=Turn("test"), generations_this_call=5)
     assert output == [
-        "",
-        "",
-        "",
-        "",
-        "",
-    ], "generators.test.Blank with generations_this_call=5 should return five empty strings"
+        Turn(""),
+        Turn(""),
+        Turn(""),
+        Turn(""),
+        Turn(""),
+    ], "generators.test.Blank with generations_this_call=5 should return five Turns with empty text"
 
 
 def test_generators_test_repeat():
     g = Repeat(DEFAULT_GENERATOR_NAME)
-    output = g.generate(prompt=DEFAULT_PROMPT_TEXT)
+    output = g.generate(prompt=Turn(DEFAULT_PROMPT_TEXT))
     assert output == [
-        DEFAULT_PROMPT_TEXT
-    ], "generators.test.Repeat should send back a list of the posed prompt string"
+        Turn(DEFAULT_PROMPT_TEXT)
+    ], "generators.test.Repeat should send back a list of the posed prompt Turn"
 
 
 def test_generators_test_single_one():
     g = Single(DEFAULT_GENERATOR_NAME)
-    output = g.generate(prompt="test")
+    output = g.generate(prompt=Turn("test"))
     assert isinstance(
         output, list
     ), "Single generator .generate() should send back a list"
     assert (
         len(output) == 1
-    ), "Single.generate() without generations_this_call should send a list of one string"
+    ), "Single.generate() without generations_this_call should send a list of length one"
     assert isinstance(
-        output[0], str
-    ), "Single generator output list should contain strings"
+        output[0], Turn
+    ), "Single generator output list should contain Turns"
 
-    output = g._call_model(prompt="test")
+    output = g._call_model(prompt=Turn("test"))
     assert isinstance(output, list), "Single generator _call_model should return a list"
     assert (
         len(output) == 1
     ), "_call_model w/ generations_this_call 1 should return a list of length 1"
     assert isinstance(
-        output[0], str
-    ), "Single generator output list should contain strings"
+        output[0], Turn
+    ), "Single generator output list should contain Turns"
 
 
 def test_generators_test_single_many():
     random_generations = random.randint(2, 12)
     g = Single(DEFAULT_GENERATOR_NAME)
-    output = g.generate(prompt="test", generations_this_call=random_generations)
+    output = g.generate(prompt=Turn("test"), generations_this_call=random_generations)
     assert isinstance(
         output, list
     ), "Single generator .generate() should send back a list"
@@ -75,67 +76,67 @@ def test_generators_test_single_many():
     ), "Single.generate() with generations_this_call should return equal generations"
     for i in range(0, random_generations):
         assert isinstance(
-            output[i], str
-        ), "Single generator output list should contain strings (all positions)"
+            output[i], Turn
+        ), "Single generator output list should contain Turns (all positions)"
 
 
 def test_generators_test_single_too_many():
     g = Single(DEFAULT_GENERATOR_NAME)
     with pytest.raises(ValueError):
-        output = g._call_model(prompt="test", generations_this_call=2)
+        output = g._call_model(prompt=Turn("test"), generations_this_call=2)
     assert "Single._call_model should refuse to process generations_this_call > 1"
 
 
 def test_generators_test_blank_one():
     g = Blank(DEFAULT_GENERATOR_NAME)
-    output = g.generate(prompt="test")
+    output = g.generate(prompt=Turn("test"))
     assert isinstance(
         output, list
     ), "Blank generator .generate() should send back a list"
     assert (
         len(output) == 1
     ), "Blank generator .generate() without generations_this_call should return a list of length 1"
     assert isinstance(
-        output[0], str
-    ), "Blank generator output list should contain strings"
-    assert (
-        output[0] == ""
-    ), "Blank generator .generate() output list should contain strings"
+        output[0], Turn
+    ), "Blank generator output list should contain Turns"
+    assert output[0] == Turn(
+        ""
+    ), "Blank generator .generate() output list should contain Turns"
 
 
 def test_generators_test_blank_many():
     g = Blank(DEFAULT_GENERATOR_NAME)
-    output = g.generate(prompt="test", generations_this_call=2)
+    output = g.generate(prompt=Turn("test"), generations_this_call=2)
     assert isinstance(
         output, list
     ), "Blank generator .generate() should send back a list"
     assert (
         len(output) == 2
     ), "Blank generator .generate() w/ generations_this_call=2 should return a list of length 2"
     assert isinstance(
-        output[0], str
-    ), "Blank generator output list should contain strings (first position)"
+        output[0], Turn
+    ), "Blank generator output list should contain Turnd (first position)"
     assert isinstance(
-        output[1], str
-    ), "Blank generator output list should contain strings (second position)"
-    assert (
-        output[0] == ""
-    ), "Blank generator .generate() output list should contain strings (first position)"
-    assert (
-        output[1] == ""
-    ), "Blank generator .generate() output list should contain strings (second position)"
+        output[1], Turn
+    ), "Blank generator output list should contain Turns (second position)"
+    assert output[0] == Turn(
+        ""
+    ), "Blank generator .generate() output list should contain Turns (first position)"
+    assert output[1] == Turn(
+        ""
+    ), "Blank generator .generate() output list should contain Turns (second position)"
 
 
 def test_parallel_requests():
     _config.system.parallel_requests = 2
 
     g = _plugins.load_plugin("generators.test.Lipsum")
-    result = g.generate(prompt="this is a test", generations_this_call=3)
+    result = g.generate(prompt=Turn("this is a test"), generations_this_call=3)
     assert isinstance(result, list), "Generator generate() should return a list"
     assert len(result) == 3, "Generator should return 3 results as requested"
     assert all(
-        isinstance(item, str) for item in result
-    ), "All items in the generate result should be strings"
+        isinstance(item, Turn) for item in result
+    ), "All items in the generate result should be Turns"
     assert all(
         len(item) > 0 for item in result
     ), "All generated strings should be non-empty"