diff --git a/.flake8 b/.flake8 index 8fe327d5..504e3af4 100644 --- a/.flake8 +++ b/.flake8 @@ -11,4 +11,6 @@ ignore = D105, D401, E302, W503 exclude = .git, __pycache__, + .project, + docs/src ignore_decorators=inherit_docs diff --git a/README.md b/README.md index d12db821..389b2c07 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,8 @@ docker container that only needs to adhere to an I/O structure of your choice (by default, in the form of `json`-files.) If you are interested in how to use the framework for a -lab course of your own, please consult our -[teaching concept](https://www.algobattle.org/docs/teaching_concept/english). +lab course of your own, please consult the teaching concept in +the [documentation](https://www.algobattle.org/docs/). # Installation and Usage This project is developed and tested on all major operating systems. diff --git a/docs/advanced/config.md b/docs/advanced/config.md index cbd19336..1a0c1e7c 100644 --- a/docs/advanced/config.md +++ b/docs/advanced/config.md @@ -1,6 +1,6 @@ # Settings -You can configure Algobattle in a lot of different ways, so it does exactly what you want it to do +You can configure Algobattle in a lot of different ways, so that it does exactly what you want it to do. !!! question "Unsure about TOML syntax?" TOML syntax can be a bit confusing if you're unfamiliar with it, a full explanation of it can be found on diff --git a/docs/advanced/problems.md b/docs/advanced/problems.md new file mode 100644 index 00000000..7b8b7a12 --- /dev/null +++ b/docs/advanced/problems.md @@ -0,0 +1,143 @@ +# Problems + +If you're currently a student in an Algobattle lab course, you've probably wondered how exactly the problems work, found +weird behaviour that you think might be a bug, or wanted to find out the exact file format it will accept. This page +teaches you how to get all that info from just the files you've already got in your Algobattle project folder! + +!!! note + This page goes over the anatomy of an Algobattle problem and how you can get what you're looking for from your + project folder. This means it's mainly aimed at students who are trying to get more familiar with the course or a + particular problem they're dealing with and course instructors who want to get an overview over how they work. + If you instead are a course instructor looking to make a problem from scratch the + [instructor tutorial](../instructor/problem_basic.md) has more detailed info on that process. + +## Overall Structure + +A typical problem file looks something like this + +```py title="problem.py" +{!> problem/problem.py !} +``` + +What exactly the problems we discuss here are about is not important for this guide. If you're still curious you can +find them all in our +[Algobattle problems](https://github.com/Benezivas/algobattle-problems/) repo with +additional explanations. We will now go through this file and explain what each section does. + +## The Instance Class + +This class defines what each instance of the problem will look like. It both tells you what your generator needs to +create and what input your solver will receive. At the top of the class you will find a block of attribute definitions. +In our case this is only a single line. + +```py hl_lines="1 8" +{!> problem/instance.py !} +``` + +This tells us that each instance's json file contains a single key, `job_lengths`, which maps to a list of Timespans. +As we can see from the type alias above the class definition, a Timespan just is an integer between 0 and +(2^64^ - 1) / 5. What that means is that if you're programming your generator you must ensure to not output any numbers +that do not fall in this range, and when implementing your solver you can safely assume that all inputs you will receive +are in it. + +### Instance Size + +The instance size is defined by the `size` property. + +```py hl_lines="10-12" +{!> problem/instance.py !} +``` + +!!! note + You should not include a `size` key in your instances. It will be computed from other attributes of the instance. + +### Additional Validation + +Some problem instances, like this one for the Hikers problem, also include a `validate_instance` method. + +```py +class HikersInstance(InstanceModel): + """The Hikers instance class.""" + + hikers: list[tuple[u64, u64]] + + @property + def size(self) -> int: + """The instance size is the number of hikers.""" + return len(self.hikers) + + def validate_instance(self) -> None: + super().validate_instance() + if any(min_size > max_size for min_size, max_size in self.hikers): + raise ValidationError("One hiker's minimum group size is larger than their maximum group size.") +``` + +This method contains further code that validates which inputs are allowable and which aren't. If you generate an +instance that causes this method to raise an error, your instance will be considered invalid, and you will lose the +fight. + +## The Solution Class + +This class is very similar to the instance class, except it specifies what solutions look like. In our case we again +have a single attribute and thus the solutions contain only a single key. + +```py hl_lines="2 8" +{!> problem/solution.py !} +``` + +This time we not only use an alias to specify the allowable range of integer values, but also the `SizeLen` marker which +means that the number of `assignments` must be exactly the same as the instance's size. + +### Solution Score + +Most solutions also have a `score` method. This tells Algobattle what the goal of this problem is and how to weigh +different solutions. + +```py hl_lines="10-15" +{!> problem/solution.py !} +``` + +The decorator at the top can either be `maximize` or `minimize` and tells us if bigger or smaller score values are +considered to be better. The function then computes some non-negative real number based on the instance and solution, +this will be this solutions' score. Each fight in a battle will receive a single score, that will be calculated by +comparing the solution score of the generator's and solver's solutions. + +In our example the score just is the longest time a machine takes to complete all jobs. If in the generator's solution +all machines complete their jobs in 10 units, but the solver's solution takes 12, the fight's score will be +approximately 0.83. + +### Additional Validation + +Just like instances can undergo extra validation, so can solutions. They use the `validate_solution` method for this. + +```py hl_lines="6-13" +class Solution(SolutionModel[HikersInstance]): + """A solution to a Hikers problem.""" + + assignments: dict[Hiker, u64] + + def validate_solution(self, instance: HikersInstance, role: Role) -> None: + super().validate_solution(instance, role) + group_sizes = Counter(self.assignments.values()) + + for hiker, group in self.assignments.items(): + min_size, max_size = instance.hikers[hiker] + if not (min_size <= group_sizes[group] <= max_size): + raise ValidationError("A Hiker is not happy with their assignment!") + + @maximize + def score(self, instance: HikersInstance, role: Role) -> float: + return len(self.assignments) +``` + +## The Problem Constructor + +The last part of the problem file actually creates the problem. + +```py hl_lines="36-41" +{!> problem/problem.py !} +``` + +It contains the name of the problem and references to the two classes we discussed. It also specifies what the smallest +reasonable size of this problem is. In our case an instance should contain at least one job for each machine, so the +minimum size of this problem is 5. diff --git a/docs/index.md b/docs/index.md index 5f7da10a..31141d8c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -13,10 +13,24 @@ The teams then battle against each other, generating instances for other teams, generated for them. Each team then is evaluated on its performance and is awarded points. -## Usage +## Where to start reading -The best place to start learning more about Algobattle is by reading through [the tutorial](tutorial/index.md). +The Algobattle documentation is aimed at two different groups of people: students participating in lab courses, and +instructors running them. In order to keep everything short and sweet we've structured our documentation so that +everyone can easily focus just on what they need to learn about. +For everyone, the best place to start learning more about Algobattle is by reading through +[the tutorial](tutorial/index.md). It contains everything you need to know to use the framework and start working with +it. Students won't necessarily need anything further to participate in the course, but may later run into things they +can best look up in the [advanced section](advanced/index.md). + +After finishing the tutorial we then recommend instructors to go through the topics in the +[instructor's corner](instructor/index.md). This will get you up to speed to run an Algobattle lab course on your own. + +!!! tip "Just want a broad overview?" + If you're not yet interested in reading all the nitty-gritty and just want a basic idea of how such a Lab course + works to decide if you want to run one yourself, the [teaching concept](instructor/teaching_english.md) is ideal + for you! ## Requirements diff --git a/docs/instructor/index.md b/docs/instructor/index.md new file mode 100644 index 00000000..bfc8d4ab --- /dev/null +++ b/docs/instructor/index.md @@ -0,0 +1,18 @@ +# Instructor's Corner + +This section is all about the stuff behind the scenes. It covers how you can structure the course and detailed info on +how to interact with the framework. Individual pages are largely independent of each other and can be read in sequence +or used as a reference as needed. + +## Overview + +- The [teaching concept](teaching_english.md) provides a high level overview of the project, what a full course looks + like, and what our goals are. + +- The problem pages discuss how to create your own problems for students to solve. The [intro](problem/intro.md) and + [basic section](problem/problem_file.md) cover everything you need to know to get started and work with most + types of problems. However, the framework is incredibly versatile and lets you create much more complex and out of + the box problems than you might initially think. The following pages dive into more detail on everything else. + +- [Battle types](battles.md) discusses how you can customize the Algobattle match process itself by writing your own + battle types. diff --git a/docs/instructor/problem/advanced.md b/docs/instructor/problem/advanced.md new file mode 100644 index 00000000..26327851 --- /dev/null +++ b/docs/instructor/problem/advanced.md @@ -0,0 +1,374 @@ +# Advanced Features + +This page is a loose collection of various more advanced features of the problem creation process. + +## Submodels + +So far we've just used tuples to group multiple pieces of data together. For example, we defined an edge as just a tuple +of two vertices. This works great for very simple types when it's clear what each element of the tuple means, but can +become very confusing quickly. Let's say we want to define a problem where rectangles are placed in a 2D coordinate +system. These are then defined by four integers: width, height, and x and y position. We could now define the instances +like this + +```py +class Instance(InstanceModel): + + rectangles: list[tuple[int, int, int, int]] +``` + +but we, and more importantly our students, will then have to always remember the order we put those numbers in. To +prevent bugs caused by this we can also define a helper class that inherits from `BaseModel` in `algobattle.util`. +This will then not have the instance or solution specific stuff added, but will also allow us to create json validation +specifications just like in those classes. + +```py +from algobattle.util import BaseModel + + +class Rectangle(BaseModel): + + x: int + y: int + width: int + height: int + + +class Instance(InstanceModel): + + rectangles: list[Rectangle] +``` + +!!! warning + The Pydantic package also exports a class called `BaseModel` which offers similar functionality. Always + inherit from the class Algobattle provides since it includes additional settings that are important for everything + to function correctly. + +Pydantic then expects a json object at the places where you use these types with keys and values matching the attributes +found in the class. For example, a valid instance json file for the above class can look like this: + +```json title="instance.json" +{ + "rectanlges": [ + { + "x": 3, + "y": 2, + "width": 5, + "height": 2 + }, + { + "x": 0, + "height": 17 + "width": 5, + "y": -2, + } + ] +} +``` + +## External Dependencies + +You may want to import some additional python libraries to use in your problem file, such as +[networkx](https://networkx.org/) when implementing a more complicated graph problem. To do this we not only have to add +the import statement in the `problen.py` file, but also include it in the list of dependencies in the project config. + +```toml title="algobattle.toml" +[match] +problem = "Some Graph Problem" + +[problem] +dependencies = [ + "networkx", +] +``` + +When initializing the project Algobattle will then make sure that the needed libraries are installed on the user's +machine. + +## Hiding Parts of the Instance + +Sometimes we want the generator to give us some data we need to verify the instance, but don't want the solver to see +this. For example, consider the problem of finding the longest path in a graph with a vertex cover (1) of a specific +size. The best way to verify that the instance graph actually contains such a vertex cover is to have it be part of the +instance data. But we want the solver to only have access to the information that it exists, not which exact vertices +are in it. +{.annotate} + +1. A vertex cover is a subset of vertices such that all other vertices in the graph have an edge to a vertex in it. + +We can use this using the `Field` construct from Pydantic. It is a simple marker object that lets us configure various +properties of specific fields. One of these settings is the `exclude` key, which tells Pydantic to exclude this field +when serializing the object into json. It will still be parsed and validated normally when reading json data and +creating the Python object. We can use it either as the default value of the attribute, or as `Annotated[...]` metadata. + +!!! example + In this class + + ```py + from pydantic import Field + + class Instance(InstanceModel): + """An instance of the Some Example problem.""" + + normal_attribute: int + hidden: float = Field(exclude=True) + also_hidden: Annotated[str, Field(exclude=True)] + ``` + + the first attribute `normal_attribute` will be the only that is included in the output that the solver sees. All three + attributes are required to be in the instance data the generator creates and will be available on the Python object. + + +!!! tip + The `Field` specifier lets you do many more things than this! Read the excellently written + [Pydantic documentation](https://docs.pydantic.dev/latest/concepts/fields) to see more use cases. + +## Using the Algobattle Graph Classes + +Since many problems use graphs as the foundation of their instances we provide several utility classes to make working +with these easier. These classes are `DirectedGraph`, `UndirectedGraph`, `EdgeWeights`, and `VertexWeights`. Using these +classes also ensures that multiple graph problems use the same formats and students won't have to worry about changing +any boilerplate code. + +The easiest use case is to just inherit from `DirectedGraph` or `UndirectedGraph` instead of `InstanceModel`. +Your instance class will then behave the same as if it also included the `num_vertices` and `edges` keys which hold a +single number specifying the number of vertices in the graph (numbered `0` through `n-1`) and a list of tuples of such +vertices respectively. They will also ensure proper validation, with `DirectedGraph` accepting any graph and +`UndirectedGraph` accepting only those that contain no self loops and where edges are interpreted as being +directionless. Both graph's size is the number of vertices in it. + +!!! example "Reachability" + An implementation of the Reachability (1) problem's instance class might look + something like this: + {.annotate} + + 1. Given a graph and two vertices in it, is there a path between them? + + ```py + class Instance(DirectedGraph): + """An instance of a Reachability problem.""" + + start: Vertex + end: Vertex + ``` + + Which is equivalent to + + ```py + class Instance(InstanceModel): + """An instance of a Reachability problem.""" + + num_vertices: u64 + edges: Annotated[list[tuple[SizeIndex, SizeIndex]], UniqueItems] + + start: Vertex + end: Vertex + + @property + def size(self) -> int: + """A graph's size is the number of vertices in it.""" + return self.num_vertices + + ``` + +!!! tip "Associated Annotation Types" + As you can see in the example above, we also provide several types that are useful in type annotations of graph + problems such as `Vertex` or `Edge`. These are documented in more detail in the + [advanced annotations](annotations.md) section. + +If you want the problem instance to also contain additional information associated with each vertex and/or each edge +you can use the `VertexWeights` and `EdgeWeights` mix ins. These are added as an additional parent class and must be +indexed with the type of the weights you want to use. + +!!! example "Labelled Vertices and Weighted Edges" + Say your problem wants to label each vertex with the name of a city and each edge with the distance it represents. This + would be done like this: + + ```py + class Instance(DirectedGraph, VertexWeights[str], EdgeWeights[float]): + + ... + ``` + + Both are encoded as lists of the weights where the nth entry corresponds to the weight of the nth vertex or edge. + I.e. the above is equivalent to + + ```py + class Instance(DirectedGraph): + + vertex_weights: Annotated[list[str], SizeLen] + edge_weights: Annotated[list[float], EdgeLen] + + ... + ``` + +## Comparing Floats + +!!! abstract + This is a somewhat esoteric section that is not strictly needed to use Algobattle. If you're interested in the + details this is perfect for you, but the important takeaway for everyone is that we recommend everyone to use the + `LaxComp` class or `lax_comp` function when working with floats. + +We use floats to represent real numbers, but these are limited to a certain precision (64 bits). This can lead to +annoying corner cases, finicky bugs, and teams being encouraged to put more energy into running into these than actually +solving the problem. For example, the equality `0.1 + 0.1 + 0.1 == 0.3` does not actually hold when using floats. +If a problem instance makes use of floats students can then use these inaccuracies to specifically craft instances that +can only be solved when carefully keeping track of your exact arithmetical operations and the inaccuracies they +introduce. Most of the time this is not actually what we want the focus of a problem to be on, so we'd rather students +just ignore these corner cases when working with floats and treat them as close to actual real numbers as possible. + +Normally we do this by never comparing strict equality between float values and instead just checking if they are close +"enough" for our use case. This is not enough for us since teams would then just create corner case instances that rely +on the exact error bound we use. The solution then is to allow the solving team to introduce bigger errors than the +generating team. That means that the generating team cannot create instances right at the edge of precision errors since +the solver's solutions will be validated with bigger allowable errors. + +This sounds complicated, but we've already implemented the hard bits in a utility class and function for you. All you +need to keep in mind is that whenever you would be doing a comparison involving equality (`==`, `<=`, or `>=`) to +instead use `LaxComp` or `lax_comp` from the `algobattle.types` module. The first acts as a wrapper that will then +safely perform these comparisons and the second performs the comparison immediately. + +!!! example + Here's several usage examples assuming the `role` variable contains the role of the team we're currently validating + something for: + + - `#!py LaxComp(0.1 + 0.1 + 0.1, role) == 0.3` + - `#!py LaxComp(0.2, role) <= 0.3` + - `#!py lax_comp(0.1 + 0.1 + 0.1, "==", 0.3 role)` + - `#!py lax_comp(0.300001, ">=", 0.3 role)` + +The margins for error these introduce are absolutely tiny for all normal applications, about 10^14^ times smaller than +the values that are being compared, so they can be safely ignored by the application logic. But they are big enough to +cover any normal errors introduces by float arithmetic and thus make it safe to naively use them as though they +represent actual real numbers. + +## Problems Without Witnesses + +Most problems require the generator to not only create an instance, but also provide a solution for it. But we can also +create problems that expect only the instance in the generator's output. To do this, just set the corresponding argument +in the Problem constructor to `#!py False`. + +```py hl_lines="6" +Problem( + name="Instance Only Problem", + min_size=17, + instance_cls=Instance, + solution_cls=Solution, + with_solution=False, +) +``` + +## Custom Scoring Functions + +In a match each fight (one execution of the generator followed by the other team's solver) is assigned a score. This is +a number in [0, 1] that indicates how well the solver did. Normally it is computed by just dividing the solver's +solution score by the generator's (or vice versa, if the goal is to minimize the score). This matches the usual notion +of approximation ratios. + +!!! example + When using the Independent Set (1) problem the generator created a solution with an independent set of size 5. The + solver found one of size 4. Since the goal here is to maximize the solution size the score of this fight would be + `0.8`. But for Vertex Cover (2) the objective is to find the smallest vertex cover, so if the solver found one of + size 20 and the generator of size 17, the overall score would be roughly `1.18`. + {.annotate} + + 1. Given a graph, find the biggest subset of its vertices that have no edge between any pair of them. + 2. Given a graph, find the smalles subset of vertices such that every other vertex has an edge to one in that set. + +But this isn't always what we want to do. Consider the problem of finding the three shortest paths between two given +vertices in a graph, formalized like this: + +```py +class Instance(DirectedGraph): + """Instances of the 3 Shortest Paths problem.""" + + start: Vertex + end: Vertex + + +class Solution(SolutionModel): + """Solutions of the 3 Shortest Paths problem.""" + + paths: tuple[Path, Path, Path] # (2)! + + @minimize + def score(self, role: Role) -> float: + ... + +Problem( + name="3 Shortest Paths", + min_size=5, + instance_cls=Instance, + solution_cls=Solution, +) +``` + +1. For clarity, we omit the imports here. +2. You'd need additional validation logic here, but we now want to focus on just the score function. + +How do we best implement the `score` function? We could just add the lengths of all the paths, or just pick the length +of the shortest or longest one. But really, what we want is for the final score to not just compare a single number +for each solution, but each path individually. + +We can achieve this by providing a custom scoring function to the Problem constructor. This just is a function that +directly receives the instance and both solutions and returns a float in [0, 1]. When we do this, we can drop the +`score` method in the Solution class entirely. + +```py hl_lines="14-23 30" +class Instance(DirectedGraph): + """Instances of the 3 Shortest Paths problem.""" + + start: Vertex + end: Vertex + + +class Solution(SolutionModel): + """Solutions of the 3 Shortest Paths problem.""" + + paths: tuple[Path, Path, Path] + + +def compare_each_path( + instance: Instance, + generator_solution: Solution, + solver_solution: Solution +) -> float: + gen_lens = sorted(len(path) for path in generator_solution.paths) # (1)! + sol_lens = sorted(len(path) for path in solver_solution.paths) + ratios = [len(gen) / len(sol) for gen, sol in zip(gen_lens, sol_lens)] # (2)! + ratios = [min(1, max(0, num)) for num in ratios] # (3)! + return sum(ratios) / 3 # (4)! + +Problem( + name="3 Shortest Paths", + min_size=5, + instance_cls=Instance, + solution_cls=Solution, + score_function=compare_each_path, +) +``` + +1. Get each path's length and sort them. +2. Compute the ratio of each corresponding pair of paths. +3. Clamp the ratios to be in [0, 1]. +4. Return the average of the ratios. + +## Test Instances + +When running `algobattle test` the CLI tool first tries to build and run the generator and then the solver. But in order +to be able to test run the solver, we need to provide it with an input instance. This means that if your generator does +not run successfully you also cannot test your solver. To prevent this issue, we can provide a simple test instance when +defining the problem. It will then be passed to the solver instead. + +```py hl_lines="6" +Problem( + name="Pairsum", + min_size=4, + instance_cls=Instance, + solution_cls=Solution, + test_instance=Instance(numbers=[1, 2, 3, 4]), +) +``` + +!!! attention "Make sure to pass a valid instance" + This instance will not undergo the usual validation step and does not come with a solution. This means you can + accidentally provide a test instance which can't actually be solved correctly. diff --git a/docs/instructor/problem/annotations.md b/docs/instructor/problem/annotations.md new file mode 100644 index 00000000..352f27bd --- /dev/null +++ b/docs/instructor/problem/annotations.md @@ -0,0 +1,345 @@ +# Advanced Type Annotations + +We've already seen how you can use type annotations to declare what shape the I/O data should have and perform basic +validation. This page will go over more advanced usages of type annotations that Algobattle and Pydantic provide for us. + +!!! note + While validation via type annotations can be very useful and much faster than plain python methods, they are not + necessary for most problems. + Everything covered here can also be done with validation methods (`validate_instance` / `validate_solution`). If + you're more comfortable with those rather than type annotations, feel free to use them instead. + +## Type Aliases + +!!! note + This section is not specific to Algobattle and just covers general Python techniques, feel free to skip it if you're + already familiar with it. + +When using complicated types our annotations can get very complicated quickly. We can simplify the code by defining +type aliases, which basically just are variables but for types. For example, consider this class + +```py +class Example(InstanceModel): + + edges: list[tuple[int, int]] + matchings: list[set[tuple[int, int]]] +``` + +Its attributes are rather terse and hard to understand what exactly a list of sets of tuples of integers is supposed +to represent. This can be simplified by creating a couple of type aliases. The syntax used depends a bit on your Python +version and how explicit you want (and have) to be, but they all do the same thing. + +=== "3.11 implicit" + + ```py + Edge = tuple[int, int] + Matching = set[Edge] + + class Example(InstanceModel): + + edges: list[Edge] + matchings: list[Matching] + ``` + +=== "3.11 explicit" + + ```py + Edge: TypeAlias = tuple[int, int] + Matching: TypeAlias = set[Edge] + + class Example(InstanceModel): + + edges: list[Edge] + matchings: list[Matching] + ``` + +=== ">= 3.12" + + ```py + type Edge = tuple[int, int] + type Matching = set[Edge] + + class Example(InstanceModel): + + edges: list[Edge] + matchings: list[Matching] + ``` + +This is particularly useful if you want to reuse a type definition, or one is very long. But it's also great to tell +others reading your code what you actually intended each piece to mean by just giving things more descriptive names. +For example, the `Vertex` type in `algobattle.types` actually just is a descriptive alias for the more general +`SizeIndex`. + +## Forward References + +!!! note + This section is not specific to Algobattle and just covers general Python techniques, feel free to skip it if you're + already familiar with it. + +Python files are executed from top to bottom and this also includes type hints. This means that you cannot use types +and classes that you define later in a type hint. In practice, this is not something you very often want to do in +problem definitions anyway, but it's worth keeping in mind. For example, let's say we want to specify a type which +emulates the way paths in a file system work. That is, it can either just be the name of a file, or correspond to +folder containing more files and folders. Ideally, we'd want just recursively define it like this: + +=== "3.11 implicit" + + ```py + Path = str | dict[str, Path] + ``` + +=== "3.11 explicit" + + ```py + Path: TypeAlias = str | dict[str, Path] + ``` + +=== ">= 3.12" + + ```py + type Path = str | dict[str, Path] + ``` + +But at the time that the `Path` on the right-hand side gets evaluated it will be an undefined variable and thus throw +an error. We can solve that by wrapping the entire expression in a string. The Python interpreter will then not evaluate +the individual variables, but type checkers and Pydantic will still interpret them correctly. The problem then is that +if we use the implicit version type checkers think that we just mean some meaningless string and not a type hint. +Because of this we actually have to use the explicit version when quoting forward references. + +=== "3.11 explicit" + + ```py + Path: TypeAlias = "str | dict[str, Path]" + ``` + +=== ">= 3.12" + + ```py + type Path = "str | dict[str, Path]" + ``` + +!!! info + The `type` syntax introduced in 3.12 actually allows you to write this specific example without the quotes. But it + only allows for forward references to the type you're defining itself to be unquoted, all other uses of forward + references still need to be quoted. + +You can also use quoted forward references in any other place you'd use a type hint, though for the types used in a +problem definition we can usually prevent them altogether by just reordering the code. + +```py +class Example(InstanceModel): + + some_attr: "CoolNumber" + +CoolNumber = int +``` + +## Annotated Metadata + +!!! note "Type Hints and Annotations" + Usually _type hint_ and _type annotation_ are used interchangeably, they just refer to the thing after the colon + following an attribute name. Since this section also deals with the `Annotated[...]` type construct we will use + type hints here when talking about the general construct to differentiate it from this specific object. + +In the basic tutorial we've already seen that we can add validation to a field using `Annotated[...]` metadata. This is +a very powerful construct that is heavily used by Algobattle and Pydantic, so we'll take a deeper look at it now. In +Python type hints are not only used by linters and type checkers to make sure your code does what you want it to, +but can also be examined at runtime. This is how Pydantic (and thus Algobattle) knows what you want the json files to +look like, it sees an attribute that's been marked as an `int`, so it will expect an integer at that place of the json +file. This is a really clever method because it will automatically validate the json without us explicitly telling us +what it should do, it just gets all the info it needs from the type hints. + +But sometimes we would want to tell the validator more than we can express in a type hint. For example, we might want to +only allow positive numbers, but Python does not have a type specifically for that. In earlier versions of Pydantic you +would then specify this using its `Field` specifier like this + +```py +class Example(InstanceModel): + + positive_int: int = Field(gt=0) + +``` + +where the `gt` key tells Pydantic that it should validate this field as being greater than 0. This works great when you +want to have this behaviour on only a single attribute, but leads to a lot of code duplication when you want it in more +places and lets you forget it easily. + +The idea behind `Annotated[...]` is that it lets us annotate a Python type with some additional metadata that is +irrelevant for type checkers, but tells other tools like Pydantic what they should do. It receives at least two +arguments, the first of which must be a type and all the others are arbitrary metadata. This lets easily specify how +several fields should be validated with a single `Field`. + +```py +PositiveInt = Annotated[int, Field(gt=0)] + +class Example(InstanceModel): + + first: PositiveInt + second: PositiveInt + third: PositiveInt + fourth: PositiveInt + +``` + +The Python standard library `annotated_types` also contains a collection of basic metadata types such as `Gt`, `Ge`, +`Lt`, `Le` that Pydantic will also interpret the same way as a `Field` with the corresponding key set. + +!!! example + In this class, all attributes will be validated as an integer between 3 and 17 inclusive. + + ```py + class Example(InstanceModel): + + first: int = Field(ge=3, lt=18) + second: Annotated[int, Field(ge=3, lt=18)] + third: Annotated[int, Ge(3), Lt(18)] + fourth: Annotated[int, Interval(ge=3, lt=18)] + ``` + + +The `algobattle.types` module also contains versions of these that behave identically for these use cases. We will later +see some capabilities of the Algobattle metadata that neither other option can do, but for most problems you can use +whichever method you prefer. + +The full list of available `Field` keys can be found in the +[Pydantic documentation](https://docs.pydantic.dev/latest/concepts/fields). The available `algobattle.types` metadata +is: + +- `Gt`, `Ge`, `Lt`, `Le`, and `Interval`: All specify a constraint on numeric data. The first four provide the + corresponding inequality and `Interval` lets you group multiple of them together by using its keyword arguments. +- `MultipleOf`: Specifies that a numeric value is a multiple of some value. E.g. a field using + `Annotated[int, MultipleOf(2)]` validates that the number in it is even. +- `MinLen`, `MaxLen`, and `Len`: Specifies that some collection's length has the corresponding property. `Len` again + serves to group the other two into a single object. E.g. `Annotated[set, MinLen(17)]` allows only sets that have + at least 17 elements. +- `UniqueItems`: Specifies that a collection contains no duplicate elements. E.g. `Annotated[list, UniqueItems]` + validates that the list contains no element twice. +- `In`: Specifies that some value is contained in a collection. E.g. `Annotated[int, In({1, 3, 17, 95})]` + allows only 1, 3, 17, or 95. +- `IndexInto`: Specifies that a value is a valid index into some list. E.g. + `Annotated[int, IndexInto(["a", "b", "c"])]` only allows numbers between 0 and 2. + +## Attribute References + +The `Field` specifiers and default metadata options cover a wide variety of use cases, but there are some validations +that cannot be done with it. For example, consider the simple problem of finding the biggest number in a list. We +can easily validate that the number actually is an element of the list with a `validate_solution` method like this: + +```py +class Instance(InstanceModel): + + numbers: list[int] + + +class Solution(InstanceModel): + + biggest: int + + def validate_solution(self, instance: Instance, role: Role) -> None + if self.biggest not in instance.numbers: + raise ValidationError("The given number is not in the instance") +``` + +But we cannot do this with the `In` annotation metadata since there we need to provide the list of items to check +against at the time we write the type hint, but we only actually get that list when we validate the solution. The +`InstanceRef` and `SolutionRef` types in the `algobattle.problem` module fix this issue. They can be used to tell +Algobattle that we do not actually want to compare against a value we have right now, but with a value that we know +will be found in the instance or solution. Our example problem then becomes simplified to this. + +```py +class Instance(InstanceModel): + + numbers: list[int] + + +class Solution(InstanceModel): + + biggest: Annotated[int, In(InstanceRef.numbers)] +``` + +!!! warning + We cannot statically ensure that the attributes you reference actually exist on the instance or solution. This + means that if you e.g. have a typo or change a definition without updating a reference to it, the validation step + will throw an error at runtime even though type checkers and linters do not raise any warnings. + + You also need to make sure you always use these in contexts where the referred to value actually makes sense. For + example, referring to an attribute of a solution when validating an instance or self-referential attributes can + lead to issues during validation. Especially in the latter case we also cannot guarantee that an error is raised in + cases where the references do not behave in the way you intended and instead will just fail silently. + +??? info "Performance" + Due to implementation details references to the object that is being validated itself (i.e. `SolutionRef` in a + solution or `InstanceRef` in an instance) will lead to two separate invocations of Pydantic's validation logic. + This is perfectly fine in basically all use cases, but when you implement very slow custom logic using it, are + validating truly massive amounts of data (several gigabytes at a time) it can lead to slowdowns. + +## Further Pydantic Features + +There are many more Pydantic features that can be very useful when designing problems. They are all explained very well +in their official documentation. In particular, +[annotated validators](https://docs.pydantic.dev/latest/concepts/validators/#annotated-validators), +[model validators](https://docs.pydantic.dev/latest/concepts/validators/#annotated-validators), +[field specifiers](https://docs.pydantic.dev/latest/concepts/fields/), +[tagged unions](https://docs.pydantic.dev/latest/concepts/unions/#discriminated-unions), +and [custom types](https://docs.pydantic.dev/latest/concepts/types/#custom-types) are very useful for Algobattle +problems. + +### Attribute Reference Validators + +!!! abstract + This is an advanced feature and will make most sense to you if you already understand + [annotated validators](https://docs.pydantic.dev/latest/concepts/validators/#annotated-validators). + +Similar to the `algobattle.types` versions of metadata annotations, `algobattle.problem` also contains the +`AttributeReferenceValidator`. It functions just like a Pydantic `AfterValidator` (and is implemented using it), but +the validation function also receives the value of a referenced attribute. + +!!! example + + If we wanted to confirm that a line of text is indented by as many spaces as are given in the instance we can + create this annotated type: + + ```py + def check_indentation(val: str, indent_level: int) -> str: + if not val.startswith(" " * indent_level): + raise ValueError + + IndentedLine = Annotated[str, AttributeReferenceValidator(check_indentation, InstanceRef.indentation)] + ``` + +### Validation Context + +!!! abstract + This is an advanced feature and will make most sense to you if you already understand + [validation context](https://docs.pydantic.dev/latest/concepts/validators/#validation-context). + +Algobattle will include certain useful data in the validation context. The full list of available keys is: + +`max_size` +: Contains the maximum allowed instance size of the current fight. Will always be present. + + !!! tip + + Keep in mind that this is a different value from the current instance's size. You usually want to use the latter + when validating data. + +`role` +: Contains the role of the program whose output is currently being validated. Will always be present. + +`instance` +: Contains the current instance. Optional key. + +`solution` +: Contains the current solution. Optional key. + +`self` +: Contains the object that is currently being validated. Optional key. + +!!! warning + Due to implementation details we sometimes need to validate data multiple times, with intermediate runs only + receiving partial validation contexts. Because of this always make sure that you check if the keys you are + accessing are currently present and do not raise an error if they aren't. + + When using the references to the object that is currently being validated keep in mind that you are accessing an + intermediate representation of it that is not guaranteed to have the properties enforced by any other functions + that rely on references to the object itself. diff --git a/docs/instructor/problem/example.md b/docs/instructor/problem/example.md new file mode 100644 index 00000000..f8489a7a --- /dev/null +++ b/docs/instructor/problem/example.md @@ -0,0 +1,593 @@ +# A Complex Example +## The 2D Knapsack Problem +In this section, we implement the so called 2D Knapsack Problem in its entirety, +starting from scratch and ending with a packaged problem archive that +can be handed out to others. + +!!! abstract "Usage of Complex Features" + In parts of this we will use rather advanced features of Algobattle. We recommend looking up the more detailed + explanations of anything you're unsure about in its corresponding section. + +To start, let us define the problem. We are given a two-dimensional, rectangular +space with an integer length and width, each of size at least one. We now +want to pack a number of smaller rectangles into this space, such that no two +pieces overlap and such that as much space is covered as possible. Each +piece may be rotated by 90 degrees. + +An instance for this problem thus consists of the dimensions of the knapsack as +well as a limited set of rectangular items. A solution for this problem then +describes where which piece should be placed and if it should be rotated by 90 +degrees before being placed. The size of the solution is then the total area +covered. + +## Starting Off +As in the previous section, we use the `algobattle` cli to construct a dummy +problem folder for us. Since we are interested in later writing a dummy +generator and dummy solver in python, we let the cli generate a stub of them +for us as well as follows: + +```console +~ algobattle init --new -p "2D Knapsack" -l python +Created a new problem file at 2D Knapsack/problem.py +Created a python generator in 2D Knapsack/generator +Created a python solver in 2D Knapsack/solver +Initialized algobattle project in 2D Knapsack +``` + +We navigate into the newly created folder named `2D Knapsack`, which has the +following file structure: + +``` { .sh .no-copy } +. +└─ 2D Knapsack + ├─ generator/ + │ ├─ .gitignore + │ ├─ Dockerfile + │ ├─ generator.py + │ └─ pyproject.toml + ├─ results/ + ├─ solver/ + │ ├─ .gitignore + │ ├─ Dockerfile + │ ├─ pyproject.toml + │ └─ solver.py + ├─ .gitignore + ├─ problem.py + └─ algobattle.toml +``` + +Before we implement anything, we should take the time to specify exactly how +an instance should look like. This means specifying the names of each key, +their function and which values are legal for each. + +First of all, an instance should contain the dimensions of the knapsack that is +to be packed. For this, we introduce two keys `height` and `width`, and would +like each to be an integer of value at least one and at most of value `2**64`. + +Secondly, we need to describe the items that are to be packed. Each item itself +has a height and a width, which should again each be of integer size at least +one. As an added restriction, we only want to allow items that are at all able +to fit into the knapsack, so as not to allow spamming a solver with items that +can never be part of a valid solution. To spare us some headache in the validation +step of the instance, we demand each item to fit into the knapsack without any +rotation, as a sort of normalization of the instance. + +Next, we need to specify the contents of a valid solution file. This is in +principle quite simple: We are interested in a list specifying which of +the items of the instance +* should be placed in the knapsack +* at which position +* being rotated or not. + +We will be lazy and define a dictionary `packing` that maps the index of items +to a three-tuple, as described. + +## Writing a Mock Generator and Solver +Before we start writing any problem code, we write a mock generator and a mock +solver. This helps us do plausibility checks while writing the actual problem file +and is not required for the finished problem file. + +We start with filling in the generator. + +```py title="generator/generator.py" +"""Main module, will be run as the generator.""" +import json +from pathlib import Path + + +max_size = int(Path("/input/max_size.txt").read_text()) + +instance = { + 'height': 4, + 'width': 3, + 'items': [ + [1, 3], + [4, 3], + [3, 3], + [3, 2], + [1, 3] + ] +} + +solution = { + 'packing': { + 0: [0, 0, 'unrotated'], + 3: [1, 0, 'unrotated'], + 4: [1, 2, 'rotated'] + } +} + + +Path("/output/instance.json").write_text(json.dumps(instance)) +Path("/output/solution.json").write_text(json.dumps(solution)) + +``` + +We made sure that the solution is not unique for the given instance +to make sure that the framework is able to compare two different solutions. +We next fill in the solver. + +```py title="generator/solver.py" +"""Main module, will be run as the solver.""" +import json +from pathlib import Path + + +instance = json.loads(Path("/input/instance.json").read_text()) + +solution = { + 'packing': { + 1: [0, 0, 'rotated'] + } +} + + +Path("/output/solution.json").write_text(json.dumps(solution)) +``` + +We are now able to immediately test any code that we write. + +## Handling Instances +We already know that we expect three keys in any instance: A `height`, +a `width` and a list of `items`. + +///note +For brevity, the following code snippets do not include all necessary imports. +We provide the complete content of the `problem.py` at the end of this section. +/// + +Our first approach uses only very rough type annotations for our expected keys +and does a lot of the validation of the instance explicitly. + +```python +"""The 2D Knapsack problem module.""" +from algobattle.problem import Problem, InstanceModel, SolutionModel, maximize +from algobattle.util import Role, ValidationError +from algobattle.types import u64 + + +class Instance(InstanceModel): + """Instances of 2D Knapsack.""" + + height: u64 + width: u64 + + items: list[u64, u64] + + def validate_instance(self) -> None: + super().validate_instance() + if self.height < 1 or self.width < 1: + raise ValidationError("The knapsack is smaller than allowed!") + if any(item[0] < 1 or item[1] < 1 for item in self.items): + raise ValidationError("An item of the instance is smaller than 1x1!") + if any((item[0] > self.height or item[1] > self.width) for item in self.items): + raise ValidationError("An item of the instance cannot fit in the knapsack!") + + @property + def size(self) -> int: + return len(self.items) +``` + +We can clean this code up by tightening up the annotations a bit. + +```python +"""The 2D Knapsack problem module.""" +from pydantic import Field +from typing import Annotated + +from algobattle.problem import Problem, InstanceModel, SolutionModel, maximize +from algobattle.util import Role, ValidationError +from algobattle.types import u64, Interval, InstanceRef + + +item_height = Annotated[int, Interval(ge=1, le=InstanceRef.height)] +item_width = Annotated[int, Interval(ge=1, le=InstanceRef.width)] +point = tuple[item_height, item_width] + + +class Instance(InstanceModel): + """Instances of 2D Knapsack.""" + + height: u64 = Field(ge=1) + width: u64 = Field(ge=1) + + + items: list[point] + + @property + def size(self) -> int: + return len(self.items) +``` + +As you can see, we have moved all explicit checks from the `validate_instance` +method into the annotations. Do note that by using the `InstanceRef` import, +we are able to use the values of some keys to annotate other keys! + +///note +If you are not familiar with pydantics annotations, we recommend +using the [pydantic documentation](https://docs.pydantic.dev/2.4/concepts/models/) +as a reference. As you have seen in the previous iteration of the code, +they are not essential, but very helpful to reduce code clutter and potential +mistakes. +/// + +This is already everything we need to implement for the instance. The `size` +method ensures that the number of items does not exceed the allowed limit given +by the instance size. We next turn to the solutions. + +## Handling Solutions +The `packing` key is slightly more involved to construct. To +recapitulate, we would like this key to be a dictionary that maps +indices of items to a two-dimensional position and an indicator +whether they should be rotated. We use a similar approach as for the +`items` list. We additionally import the `Literal` class from the +`typing` module as well as the `SizeIndex` type alias from +`algobattle.types`. + +```python +position_height = Annotated[int, Interval(ge=0, lt=InstanceRef.height)] +position_width = Annotated[int, Interval(ge=0, lt=InstanceRef.width)] +rotation = Literal["unrotated", "rotated"] + + +class Solution(SolutionModel[Instance]): + """Solutions of 2D Knapsack.""" + + packing: dict[SizeIndex, tuple[position_height, position_width, rotation]] +``` + +This of course does not at all ensure that the given solution is valid, yet. +We did not yet check whether items overlap or extend beyond the boundaries +of the knapsack. Since these checks are arguably beyond the scope of simple +type checking, we implement these tests explicitly in the `validate_solution` +method. For convenience, we import the `itertools` library. + +```python + def validate_solution(self, instance: Instance, role: Role) -> None: + flattened_packing = [] + for index, (pos_height, pos_width, rotation) in self.packing.items(): + item_height = instance.items[index][0 if rotation == "unrotated" else 1] + item_width = instance.items[index][1 if rotation == "unrotated" else 0] + + height_endpoint = pos_height + item_height + width_endpoint = pos_width + item_width + flattened_packing.append( + (index, pos_height, height_endpoint, pos_width, width_endpoint) + ) + + if height_endpoint > instance.height or width_endpoint > instance.width: + raise ValidationError( + "Item extends the knapsack boundaries.", + detail=f"Item {index} was placed at position ({pos_height, pos_width}), extending the knapsack boundaries." + ) + + for item, other_item in itertools.combinations(flattened_packing, 2): + if item[1] < other_item[2] and item[2] > other_item[1]: + if item[3] < other_item[4] and item[4] > other_item[3]: + raise ValidationError( + "Two items overlap.", + detail=f"Items {item[0]} and {other_item[0]} overlap." + ) +``` + +We are almost done writing the problem class. The next step is to tell +the framework what the quality of a solution is, i.e. which values it +should compare when given two solutions to determine which is the better one. + +For this, we overwrite the `score` method. We have access to the solution +via the `self` argument, access to the instance via the `instance` argument +and can even decide to judge the certificate solution of the generator and +a solvers solution differently, via the `role` argument, e.g. to give the +solver some additional slack. + +```python + @maximize + def score(self, instance: Instance, role: Role) -> float: + area = 0 + for index in self.packing: + area += instance.items[index][0] * instance.items[index][1] + return area +``` + +You can find the complete contents of the `problem.py` at the end of this +tutorial section. + +## Best Practice: Writing Tests +Now that we have created a problem file, it is time to see if it +does what we want it to do. The most straightforward sanity check +is to run the mock generator and solver that we have written +previously. + +///note +If you did not generate the problem folder as we did in this tutorial, +make sure that a team is entered in the `algobattle.toml` file that +utilizes the solver and generator that we wrote! +/// + +For this, we can use the `algobattle test` command. This command +builds the generators and solvers of the configured teams and +executes a single run of them at the minimum size that was configured +for the problem. + +Running this command does however produce an issue: +```console +~ algobattle test +Testing programs of team Rats +Generator built successfully +Generator didn't run successfully +Solver built successfully +Cannot test running the solver +You can find detailed error messages at results/test-2024-01-01_12-35-10.json +``` + +So what went wrong? Looking into the log files reveals the issue. + +```json +{ + "Rats": { + "generator_run": { + "type": "ValidationError", + "message": "Instance is too large.", + "detail": "Generated: 5, maximum: 1" + } + } +} +``` + +We wrote a generator and solver that run on an instance with five items, +but proclaimed in the `problem.py` that any instance with at least one +item is valid: + +```python +Problem( + name="2D Knapsack", + min_size=1, + instance_cls=Instance, + solution_cls=Solution, +) +``` + +Should we thus change our generator and solver? We do not have to, +as the `algobattle test` command allows us to run the test on a specific +instance size: + +```console +~ algobattle test --size 5 +Testing programs of team Rats +Generator built successfully +Generator ran successfully +Solver built successfully +Solver ran successfully +``` + +This tells us that the combination of our problem description +with a small, hand-crafted instance behaves as expected. It is at this +stage where most of the errors in the code come to light. You +can use the log files written into the `results` folder to assist +you in debugging your code. You may find at this stage that it does +pay off to write detailed `ValidationError` exception messages. + +Just because our single, hand-crafted test ran through, this does not +mean that our code is without any conceptual errors. Especially when +giving your problem file to other people, who will likely spend much more +time dissecting your code and descriptions to learn how to write their +own programs, many unexpected issues with your code may come to light. + +To mitigate some of the reports of illegal inputs that are nevertheless +accepted by your code, legal inputs that are rejected by your code, or worst +-- code that crashes your validation code -- it is a good idea to write a few +unittests. + +We do not want to dive into too much detail on how you could test your +code, how much coverage may be desirable and related topics, as this +goes well beyond the scope of this tutorial. Testing code is a topic +about which volumes have been written by authors who are much more +knowledgeable about the topic as we could claim to be. + +Thus, we only talk about how to best interface the problem that we +have designed, so that you can then use this knowledge to write +your own tests. We use the `unittest` module from the standard library +for this part of the tutorial. + +We create a file `tests.py` in the `2D Knapsack` folder, with generic +scaffolding. + +```py title="tests.py" +"""Tests for the 2D Knapsack problem.""" +import unittest + +from algobattle.util import Role + +from problem import Instance, Solution, ValidationError + + +class Tests(unittest.TestCase): + """Tests for the 2D Knapsack problem solution class.""" + + ... + + +if __name__ == "__main__": + unittest.main() +``` + +You can then access all additional helper methods that you may have +added to the `Instance` and `Solution` classes as you would normally do. + +If you would like to test the validation methods, i.e. `validate_instance` +and `validate_solution`, you could do so as follows. + +Assume, just for the sake of being able to give an example, that we would have +added a `validate_instance` method to the `Instance` class with the following, +rather nonsensical content: + +```python +# This method is just for demonstration purposes. +def validate_instance(self) -> None: + super().validate_instance() + if self.height != 1: + raise ValidationError("The knapsack is not of height 1!?") +``` + +This rather silly method raises a validation error whenever the +height of the knapsacks is unequal to one. + +```python +# Sample test for the validate_instance method +def test_knapsack_height_not_silly(self): + with self.assertRaises(ValidationError): + faulty_instance = Instance.model_validate({"height": 2, "width": 1, "items": [(1, 1)]}) + faulty_instance.validate_instance() + +# Sample test for the validate_solution method +def test_item_overlap(self): + instance = Instance(height=1, width=1, items=[(1, 1), (1, 1)]) + with self.assertRaises(ValidationError): + faulty_solution = Solution.model_validate({"packing": {0: (0, 0, "unrotated"), 1: (0, 0, "unrotated")}}) + faulty_solution.validate_solution(instance, Role.generator) +``` + +You can test the `size` function of the `Instance` class +and the `score` function of the `Solution` class as you would test any other +method. + +If you use the `unittest` module, you can then run these tests by +executing `python -m unittest` in the `2D Knapsack` folder. + +## Writing a Description + +We are done writing code for the problem. Now, it is a good idea +to write a description file that tells the users that should work +on the problem what it is about. This includes explaining the general +idea and, more importantly, how the expected I/O is defined. + +We recommend creating a file in the `2D Knapsack` folder named `description.md`, +as this file name is automatically picked up by the packaging step +that we will handle in the next step. + +///note +If you use the `algobattle-web` framework, the contents of this file +will be displayed to your users when they click on the respective +problem tab. +/// + +## Packaging Everything Together + +Now that our code is tested and documented, we are ready to hand it out! +For this, we can again use the `algobattle` cli, which wraps up the +`problem.py`, the `algobattle.toml` and the `description.md` into a file +that others can work on. + +```console +~ algobattle package problem +Packaged Algobattle project into /path/to/working/dir/2D Knapsack/2d_knapsack.algo +``` + +///note +The `algobattle.toml` gets truncated during the packaging step. Only the `[match]` +entries remain. +/// + +## The Completed problem.py File +This is the final content of the `problem.py` that we have created. + +```py title="problem.py" +"""The 2D Knapsack problem module.""" +import itertools +from pydantic import Field +from typing import Annotated, Literal + +from algobattle.problem import Problem, InstanceModel, SolutionModel, maximize +from algobattle.util import Role, ValidationError +from algobattle.types import u64, Interval, InstanceRef, SizeIndex + + +item_height = Annotated[int, Interval(ge=1, le=InstanceRef.height)] +item_width = Annotated[int, Interval(ge=1, le=InstanceRef.width)] +point = tuple[item_height, item_width] + + +class Instance(InstanceModel): + """Instances of 2D Knapsack.""" + + height: u64 = Field(ge=1) + width: u64 = Field(ge=1) + + items: list[point] + + @property + def size(self) -> int: + return len(self.items) + + +position_height = Annotated[int, Interval(ge=0, lt=InstanceRef.height)] +position_width = Annotated[int, Interval(ge=0, lt=InstanceRef.width)] +rotation = Literal["unrotated", "rotated"] + + +class Solution(SolutionModel[Instance]): + """Solutions of 2D Knapsack.""" + + packing: dict[SizeIndex, tuple[position_height, position_width, rotation]] + + def validate_solution(self, instance: Instance, role: Role) -> None: + flattened_packing = [] + for index, (pos_height, pos_width, rotation) in self.packing.items(): + item_height = instance.items[index][0 if rotation == "unrotated" else 1] + item_width = instance.items[index][1 if rotation == "unrotated" else 0] + + height_endpoint = pos_height + item_height + width_endpoint = pos_width + item_width + flattened_packing.append( + (index, pos_height, height_endpoint, pos_width, width_endpoint) + ) + + if height_endpoint > instance.height or width_endpoint > instance.width: + raise ValidationError( + "Item extends the knapsack boundaries.", + detail=f"Item {index} was placed at position ({pos_height, pos_width}), extending the knapsack boundaries." + ) + + for item, other_item in itertools.combinations(flattened_packing, 2): + if item[1] < other_item[2] and item[2] > other_item[1]: + if item[3] < other_item[4] and item[4] > other_item[3]: + raise ValidationError( + "Two items overlap.", + detail=f"Items {item[0]} and {other_item[0]} overlap." + ) + + @maximize + def score(self, instance: Instance, role: Role) -> float: + area = 0 + for index in self.packing: + area += instance.items[index][0] * instance.items[index][1] + return area + + +Problem( + name="2D Knapsack", + min_size=1, + instance_cls=Instance, + solution_cls=Solution, +) +``` \ No newline at end of file diff --git a/docs/instructor/problem/intro.md b/docs/instructor/problem/intro.md new file mode 100644 index 00000000..d180a856 --- /dev/null +++ b/docs/instructor/problem/intro.md @@ -0,0 +1,104 @@ +# Introduction + +!!! abstract "Brush up on the basics" + This page assumes you're already familiar with the basics of how Algobattle works. If that's not you yet, you can + read up on it in the [student tutorial](../../tutorial/getting_started.md). + +In each Algobattle course the student teams are given a few problems that they will then need to solve. This means that +we need to first come up with what those problems are and then how to tell Algobattle about them. + +## Using a Prebuilt Problem + +The fastest way to get up and running is to use one of the problems we've already developed in the +[Algobattle Problems](https://github.com/Benezivas/algobattle-problems) repository. These contain everything you need to +use them in a course and have been tested already. But of course half the fun is coming up with your own ideas and +tailoring the problems to your students! + +## Coming up with a Problem Idea + +Before we start to write any code, we need to come up with the abstract problem that students should solve. +For this, it may be easiest to first review what a problem actually is. Abstractly it's just a specification of +instances and solutions, and a mapping of instances to valid solutions. But of course, that doesn't really tell us much. +What's more helpful is to look at some examples of well known problems: + +- Satisfiability. Given a formula of propositional logic (e.g. `(A ∨ ¬C) ∧ (¬A ∨ B)`), determine if there is some truth + assignment to the variables that make it true. + +- Reachability. Given a graph and two vertices `v`, `w` in it, determine if there is some path from `v` to `w`. + +- Independent Set. Given a graph, compute the largest set of its vertices that have no edges between any two of them. + +- Subset Sum. Given a list of numbers and a target, determine if there is some subset of the list that sums to the + target. + +- Pairsum. Given a list of numbers, find two pairs `a, b` and `c, d` of numbers in it that have the same sum + `a + b = c + d`. + +We can see that each of these uses some fairly common mathematical structure (a formula, a graph, a list of numbers, +etc.) to specify instances and then specifies some goal that the solver needs to achieve. Sometimes this can be a simple +yes or no answer, but it can also be a more complicated solution. + +!!! warning "Validation vs Verification" + We use the terms _validation_ and _verification_ to refer to two distinct things. To validate an instance or + solution is to check whether it is, in principle, valid and well-formed. For example, in the Subset Sum problem + the validation process confirms that the instance is indeed a properly formatted list of numbers and a target or + that the solution is a subset of the numbers in the instance that was given. On the other hand verification actually + checks whether a solution correctly solves the given instance. + + Put shortly: `Otters` is an _invalid_ solution to `5 + 3`, `17` is _valid_ but does not pass _verification_, and + `8` is the actually correct solution that is _valid_ and passes _verification_. + +## Conceptual Requirements + +Algobattle is very flexible, so we can let our creativity run almost completely free here! But there still are some +considerations that make some types of problems more well-suited for Algobattle framework. Essentially, we are +interested in two characteristics, both which revolve around the solutions of a problem: + +1. It is fairly fast (say, at most quadratic asymptotic runtime) to check if a proposed instance or solution is valid. + This does not impact most problems since the requirements for valid instances and solutions are very direct and + easy to verify. But there are some tricky cases where a simple sounding requirement ends up being costly to + validate. + +2. The solution of an instance can be verified significantly faster than it would take to actually solve it. + In particular, the solution should contain all the information needed to determine if it is correct. Problem + solutions that are simple yes or no answers or that rely on hard to compute outside information are hard to verify + quickly and thus should be avoided. + +Both are soft requirements which you can technically ignore. This may, however, impact your match runtime significantly. +The validation and verification process does not have a built-in timeout, meaning that if you try to solve an instance +during it, the framework will not continue until this solution was found or an error is encountered. + +There is no restriction on the data encoding format that problems use. We only enforce that all data is passed in the +form of files, regardless of their encoding or even how many a single instance/solution uses. However, almost all +problems we use in our courses use a single `.json` file since this is such a universally supported file format, there +is great support built into Algobattle itself, and you can encode most things into it fairly well. For a deeper dive +into how `json` or other I/O formats work, have a look at the [I/O Section](io.md). + +## Phrasing Problems for Algobattle + +We can now look back at the problems defined above to see if they're suited for Algobattle and how we can best phrase +them. In particular, let's look at the Satisfiability and Independent Set problems. + +Satisfiability instances are very easy and fast to validate since you just need to do some basic syntax checks. +Solutions are yes or no answers, also trivial to validate. But verifying them is hard. There is currently no known +algorithm that can do this efficiently, meaning we would have to spend a lot of time solving every instance. +A better way to phrase this problem would be to instead ask for the variable assignment itself, not just whether it +exists. Then you can simply fill in the truth values and confirm that the formula evaluates to true. However, this +does not work for the negative case where no such assignment exists. + +Independent Set uses a graph for its instances, which have various common and simple to validate encodings. Algobattle +even comes with an easy-to-use one built in. Solutions are more complicated than just a yes or a no, but it still is no +issue to check that a set of vertices does not have any edges between them. But there is a different problem with +verifying the solutions, we can't easily know if a proposed independent set actually is the biggest. + +Both of these can be solved in the same way. By having each instance come with a valid solution. In the case of +Satisfiability we then guarantee that there is indeed some satisfying variable assignment. And for Independent Set we +slightly relax our problem so that we don't ask for _the biggest_ independent set but just one that is at least as big +as the one we already know exists. Since this very commonly what we want it is the default in Algobattle. We do this by +having the generator also output a solution in addition to the instance. You can change this behaviour when creating +your problem later. + +## Formalizing the Problem + +Now that we have the broad idea of how our problem should work we need to formalize it so that Algobattle can work with +it. To do this we [make a problem file](problem_file.md). diff --git a/docs/instructor/problem/io.md b/docs/instructor/problem/io.md new file mode 100644 index 00000000..d19b9500 --- /dev/null +++ b/docs/instructor/problem/io.md @@ -0,0 +1,228 @@ +# Arbitrary I/O Formats + +So far we've always encoded our problem instances and solutions as json files, but Algobattle lets you also use +whatever other data types you want for your problems. This page will go through the implementation of such a problem and +explain every step in detail. + +## Inheriting from Instance or Solution + +To define a problem that uses json encoding we just inherit from `InstanceModel` or `SolutionModel`, these are actually +subclasses of `Instance` and `Solution` that combine their functionality with the Pydantic json parsing and data +validation. This means that if we want to use our own encoding or decoding logic, we can just inherit from the base +classes instead. + +Throughout this page we will work on an example problem where instances are pictures and the task is to identify which +animal is in it. This means that we will use a json based solution and a custom encoding for the instances. +The starting point of our problem file then looks like this: + +```py title="problem.py" +from typing import Literal + +from algobattle.problem import Problem, Instance, SolutionModel + +Animal = Literal["Cat", "Dog", "Duck", "Stingray", "Albatross", "Snake"] + +class MyInstance(Instance): + """Instances of Animal Detection.""" + + ... + + @property + def size(self) -> int: + ... + + +class MySolution(SolutionModel[MyInstance]): + """Solutions of Animal Detection.""" + + found: Animal + + +Problem( + name="Animal Detection", + min_size=64, + instance_cls=MyInstance, + solution_cls=MySolution, + with_solution=False, +) +``` + +??? note "Class Names" + In this example we call our instance and solution classes different names to avoid clashes with the base classes + from `algobattle.problem`. You could instead also import them under different names or use dotted imports. + +??? note "No Generator Solution" + The way we will implement this problem is by not requiring the generator to also submit a solution. + This is just a choice we make for this particular example, you can also use custom data formats for problems that + also require a generator solution. + +## Implementing the Python Data + +Every `MyInstance` object needs to hold the info it needs to encode the instance for the solver, score it, etc. In +our example this means that we need to somehow store the image in these Python objects and implement things like the +size property or validation and scoring methods using that. We will just use a basic +[data class](https://docs.python.org/3/library/dataclasses.html) and the +[pillow](https://pillow.readthedocs.io/en/stable/) image library. + +```py title="problem.py" hl_lines="2 6 27-30" +from typing import Literal +from dataclasses import dataclass + +from algobattle.problem import Problem, Instance, SolutionModel +from algobattle.util import Role +from PIL import Image + + +Animal = Literal["Cat", "Dog", "Duck", "Stingray", "Albatross", "Snake"] + +@dataclass +class MyInstance(Instance): + """Instances of Animal Detection.""" + + image: Image.Image + + @property + def size(self) -> int: + return max(self.image.width, self.image.width) + + +class MySolution(SolutionModel[MyInstance]): + """Solutions of Animal Detection.""" + + found: Animal + + def validate_solution(self, instance: MyInstance, role: Role) -> None: + super().validate_solution(instance, role) + ... # check that the correct animal is pictured + + +Problem( + name="Animal Detection", + min_size=64, + instance_cls=MyInstance, + solution_cls=MySolution, + with_solution=False, +) +``` + +## The `Encodable` Protocol + +We now need to tell Algobattle how it should encode our instances into files and how it should decode them from the +output of a program. For the first we just implement an `encode` method that takes the location on the file system where +the data needs to end up, and the role of the team that will read this data. We can either create a new folder at the +given path and then place as many files as we want in it, or create a single file at that path. You should never create +any files that aren't rooted at the given path, or are siblings of it, etc. The path we are given will have a plain +name without any file extension, the name itself cannot be changed, but an appropriate file extension should be +added. + +```py hl_lines="11-13" +@dataclass +class MyInstance(Instance): + """Instances of Animal Detection.""" + + image: Image.Image + + @property + def size(self) -> int: + return max(self.image.width, self.image.width) + + def encode(self, target: Path, role: Role) -> None: + full_path = target.with_suffix(".png") # (1)! + self.image.save(full_path) # (2)! +``` + +1. Add the `.png` file extension +2. Write the image to the target location using pillow. + +!!! warning "Super Call" + Do not call `super().encode()` in this method. The `Instance` class's `encode` method is abstract and + will just raise an error. This is different to the validation methods. + +??? note "Role" + We can use the role argument to encode data differently based on who is going to read it. Most of the time this + argument won't be used, but it can be helpful when working with advanced battle types and problems. + +The other method we need to implement is the `decode` class method. It takes a path pointing to where the program should +have placed its output and then returns a problem instance object. It also again takes the role argument and an +additional one specifying the maximum allowable size in this fight. + +!!! info "Maximum Size" + You do not need to validate that the size of the instance actually is smaller than the maximum allowed size. This + will be done in a later step by Algobattle itself. In most use cases the `max_size` argument won't be needed, but + it can be helpful to e.g. prevent stalling in the decoding process when trying to read abnormally large files. + +```py hl_lines="15-24" +@dataclass +class MyInstance(Instance): + """Instances of Animal Detection.""" + + image: Image.Image + + @property + def size(self) -> int: + return max(self.image.width, self.image.width) + + def encode(self, target: Path, role: Role) -> None: + full_path = target.with_suffix(".png") + self.image.save(full_path) + + @classmethod + def decode(cls, source: Path, max_size: int, role: Role) -> Self: + full_path = source.with_suffix(".png") # (1)! + try: + image = Image.open(full_path) # (2)! + except FileNotFoundError: + raise EncodingError("The image file does not exist.") + except UnidentifiedImageError: + raise EncodingError("The image cannot be decoded.") + return cls(image) # (3)! +``` + +1. Add the same file extension we used when encoding the data. +2. Read the image using pillow. +3. Return a new object of the instance class. + +!!! warning "Super Call" + Do not call `super().decode()` in this method. The `Instance` class's `decode` method is abstract and + will just raise an error. This is different to the validation methods. + +??? note "Generator Solution" + If your problem does use generator solutions then you do not need to decode them in this method. The path you + receive points only to the instance data and the generator's solution will be decoded using the solution class's + `decode` method. + +When the data cannot be decoded properly or is missing you should always raise an `EncodingError` from +`algobattle.util` with appropriate error messages. This can also be in cases where you can in principle decode the +data, but it does not conform to some specification that's part of your problem. For example, when using the usual +base classes to decode json files we also apply Pydantic validation as part of this step. + +!!! warning "Decoding Solutions" + Solutions follow exactly the same encoding protocol, but additionally receive an argument on their decode method + that contains the instance this solution is for. This means that you need to implement a `decode` method like this: + + ```py + class ExampleSolution(Solution[ExampleInstance]): + + @classmethod + def decode(cls, source: Path, max_size: int, role: Role, instance: ExampleInstance) -> Self: + ... + ``` + +## Specifying the I/O Schema + +We optionally can also add a class method that specifies what exactly our instances should look like. This information +will not be used by the Algobattle framework itself, but can be used by your students. It should be a textual and +machine-readable description of what this instance's or solution's data needs to conform to. In the case of the usual +json data it is their [OpenAPI schema](https://swagger.io/specification/). What exactly this should look like depends +heavily on the data encoding techniques you are using and in many cases there simply is no reasonable schema. In those +cases it's best to just not implement this method. + +This is the signature of this method: + +```py +class ExampleInstance(Instance): + + @classmethod + def io_schema(cls) -> str | None: + ... +``` diff --git a/docs/instructor/problem/problem_file.md b/docs/instructor/problem/problem_file.md new file mode 100644 index 00000000..14321aca --- /dev/null +++ b/docs/instructor/problem/problem_file.md @@ -0,0 +1,493 @@ +# Creating a Problem File + +Algobattle uses _Problem files_ to specify all the details of a problem formally, i.e. what instances and solutions +should look like, how to score them, how to decode and encode them, etc. These are Python files and leave a lot of room +for you to write them however you like. This overview will cover the basic structure of them and common use cases, for +more advanced features refer to later sections of the problem creation guide. + +!!! example "Pairsum" + Throughout this page we will use the Pairsum problem as our working example. + +## Initializing the Project Folder + +When students develop their programs they are typically working within an Algobattle project folder created by the CLI +tool from a `.algo` file. Our goal now is to create a brand-new project folder with problem files for our new problem. +Once we're done we can then package this into a `.algo` file and distribute it to our students. + +To initialize the folder we also use the `algobattle init` command, but this time specify that we want to create a new +problem and its name. + +```console +algobattle init --new --problem "Pairsum" +``` + +This creates a new folder named `Pairsum` with the following contents: + +``` { .sh .no-copy } +. +└─ Pairsum + ├─ generator/ + │ └─ Dockerfile + ├─ results/ + ├─ solver/ + │ └─ Dockerfile + ├─ .gitignore + ├─ problem.py + └─ algobattle.toml +``` + +Let us take a look at the contents of the `problem.py` file, which is where we will implement the problem logic. + +``` { .py .title="problem.py" .no-copy } +"""The Pairsum problem module.""" +from algobattle.problem import Problem, InstanceModel, SolutionModel, maximize # (1)! +from algobattle.util import Role + + +class Instance(InstanceModel): # (2)! + """Instances of Pairsum.""" + + ... + + @property + def size(self) -> int: + ... + + +class Solution(SolutionModel[Instance]): # (3)! + """Solutions of Pairsum.""" + + ... + + @maximize + def score(self, instance: Instance, role: Role) -> float: + ... + + +Problem( # (4)! + name="Pairsum", + min_size=1, + instance_cls=Instance, + solution_cls=Solution, +) +``` + +1. These lines import the needed parts of the Algobattle framework. +2. This class specifies what instances of the problem look like. +3. This class specifies what solutions of the problem look like. +4. The `Problem` constructor ties all the information together and creates the problem. + +As you can see, the file is mostly empty now and only contains some bare-bones structure. The ellipses tell us where +we need to implement the basic problem specific code, though there also are more options for us to customize later on. + +!!! tip "Type Checking" + The Algobattle module is fully typed, but Python doesn't actually require you do include type hints in your own + code. We strongly recommend still using type hints as much as possible since they prevent many bugs and make code + easier to read. We will also make heavy use of type annotations to easily specify data formats. + +## The Instance Class + +Let's start by looking at the `Instance` class. This will specify what every instance of your problem should look like. +As you can see this class inherits from the `InstanceModel` utility class, which uses +[Pydantic](https://docs.pydantic.dev/latest/) to make instances that can easily be decoded to and from json files. + +??? info "Advanced Usage" + In the most general case they need to only inherit from the + [`Instance`](../../api/problem.md#algobattle.problem.Instance) class and implement the + [`Encodable`](../../api/util.md#algobattle.util.Encodable) protocol, but doing so manually is much more + complicated and not needed for most problems. We will see how to use these classes in the + [arbitrary i/o formats](io.md) guide. + +!!! tip "Pydantic" + Pydantic is a very powerful library with excellent support for many use cases. This can also make it harder to + understand how exactly everything works and what the "best" way of doing something is. For now, we recommend just + staying here and focusing on how Algobattle lets us use it. If you're curious and want to see how it works under the + hood you can then go back to its documentation later. + +### Instance Data + +First we need to specify what an instance actually looks like. In our case it just is a list of natural numbers. +This means we want the json file of an instance to look something like this: + +```json +{ + "numbers": [1, 3, 17, 95, 0, 24, 6] +} +``` + +I.e. it contains a single field called `numbers` which contains a list of non-negative integers. This can be copied to +the first ellipsis of the Instance class almost verbatim! + +```py +class Instance(InstanceModel): + """Instances of Pairsum.""" + + numbers: list[int] + + @property + def size(self) -> int: + ... +``` + +Here we create a Python class attribute named after the key we want in the json file, and give it a type annotation that +matches the kind of data we want at that key. If a json file contains a key that is not listed here, or if it is missing +one of the keys listed here, the framework will assume that the given instance is malformed and reject it. Pydantic +also ensures that the values at each key also match the types specified in the class! + +??? info "Unfamiliar with Python type annotations?" + Here's some more examples of type annotations and what they mean: + + - `float`: a single real number. + - `Literal["Ice Cream", "Cake", "Donuts"]`: one of `Ice Cream`, `Cake`, or `Donuts`, no other strings or any other + values are permitted. + - `tuple[int, str]`: a tuple of an integer and a string. Since json only knows lists this will look like + `[123, "Cats"]` or `[-17, "Dogs"]`, but never something like `[1, 2]` or `["Wombats", "are", "great"]`. + - `dict[str, int]`: a mapping of strings to integers, e.g. `{"Pasta": 1, "Pizza": 17, "Sushi": 5}`. + - `set[list[str]]`: a set of lists of strings. Similar to tuples, json does not support sets so they will be encoded + as plain lists, but with the requirement that no elements are duplicated and that order does not matter. + + In general, you can use most basic types on their own, with collections having the type they contain in square + brackets after them. + +!!! example "Example Instances" + Here are some valid example instances: + + - `#!json {"numbers": [1, 2, 3]}` + - `#!json {"numbers": [17, 0, 0]}` + - `#!json {"numbers": [95, 74694, 65549, 6486681, 6513232135186, 651344168]}` + + These are invalid and will be rejected automatically: + + - `#!json {"numbers": [1, 2, 3], "other": 5}` + - `#!json {}` + - `#!json {"nums": [1, 2, 3]}` + - `#!json {"numbers"` + - `#!json {"numbers": [1.5, 2, 3]}` + - `#!json {"numbers": 17}` + +### Additional Validation + +But there still is an issue with this instance definition, it says that the numbers can be any integers and not just +natural numbers. This means that `#!json {"numbers": [-1, -2, -3]}` would also be accepted! Another potential issue is +that Python allows arbitrarily large numbers in its `int` type, but many other languages make very large numbers hard to +work with. To make everything a bit fairer for different teams using different languages and not make winning a match +be based on exploiting corner case overflow bugs, we recommend also limiting the maximum size of the numbers, so they +can fit in a 64-bit integer. This can be done very easily by using the Algobattle utility types we provide in the +`algobattle.types` module. In our case we want `u64` for an unsigned 64-bit integer. + +```py +from algobattle.types import u64 # (1)! + +class Instance(InstanceModel): + """Instances of Pairsum.""" + + numbers: list[u64] + + @property + def size(self) -> int: + ... + +``` + +1. Always remember to add imports at the top of the file for everything you use from an external module. + +!!! tip "Integer Types" + The `algobattle.types` module contains predefined types for all commonly found integer types. These are `u64`, + `u32`, `u16` for unsigned integers that fit in 64, 32, and 16 bits and `i64`, `i32`, `i16` for the corresponding + signed variants. + +But there also are some properties that we cannot validate by just using one of the predefined types. The easiest way +to do that is to implement the `validate_instance` method. It will be called after all the basic properties of the +types have been validated and can then perform more complicated checks. For example, if we wanted to also add the +constraint that all the numbers are even we could add this: + +```py +from algobattle.util import ValidationError + +class Instance(InstanceModel): + """Instances of Pairsum.""" + + numbers: list[u64] + + def validate_instance(self) -> None: # (1)! + super().validate_instance() # (1)! + for number in self.numbers: + if number % 2 != 0: + raise ValidationError( + "A number in the instance is not even!", + detail=f"Odd number {number} was passed.", + ) + + @property + def size(self) -> int: + ... + +``` + +1. The `validate_instance` method takes only the instance itself as an argument, and returns nothing. +2. Always include this line at the top of this method. It will call the parent's class validation logic to make sure + that properties assured by it are actually enforced. + +If the instance is valid this method should simply return nothing, if it is invalid it needs to raise a +`ValidationError`. + +!!! note "Error Messages" + You may notice the optional `detail` argument of the `ValidationError` exception. When the logs are visible for + everyone, accidentally leaking information about parts of an instance, may reveal the strategy of a team. On + the other hand, when developing code, a team may nevertheless _want_ to see exactly what went wrong. + + The first argument will always be visible to everyone, while the `detail` field is hidden in match logs but visible + in local test runs. This means that the first argument should only contain a basic description of the general error + and detailed info should be in the `detail` argument. + +Implementing this method is entirely optional. Many simpler problems like Pairsum do not need it at all since their +properties are easily encoded in just the predefined types. + +### Instance Size + +Each instance also has a specific _size_ that is used to limit the generating team so that it actually needs to produce +_hard_ instances and not just _big_ ones. In our case the size naturally just is the length of the list of numbers. +We specify what a specific instance's size is by implementing the `size` property in the Instance class. + +```py +class Instance(InstanceModel): + """Instances of Pairsum.""" + + numbers: list[u64] + + @property + def size(self) -> int: + return len(self.numbers) + +``` + +There are many more things you can customize here, but this is all you need to know to get started. If you want to take +a deeper dive, check out the advanced problem creation pages once you've got a feeling for everything. + +## The Solution Class + +The Solution class works very similar to the Instance class. Its job is to specify what solutions look like and how to +score them. The data encoding and decoding can again be done using Pydantic: + +```py +class Solution(SolutionModel[Instance]): + """Solutions of Pairsum.""" + + indices: tuple[u64, u64, u64, u64] + + @maximize + def score(self, instance: Instance, role: Role) -> float: + ... +``` + +But note that we're actually looking for four different indices into the list in the input, not just any four numbers. +That means we need to validate that the numbers are valid indices (i.e. smaller than the length of the list) and are +all different from each other. We could again do that with a custom validation method, but we can also use some more +advanced utility types. + +```py +class Solution(SolutionModel[Instance]): + """Solutions of Pairsum.""" + + indices: Annotated[tuple[SizeIndex, SizeIndex, SizeIndex, SizeIndex], UniqueItems] + + @maximize + def score(self, instance: Instance, role: Role) -> float: + ... +``` + +The first change is to use `SizeIndex` instead of a `u64`. This ensures that the numbers are valid indices into a list +of the length of the `size` of the instance. In our case the size is defined to be exactly the length of the list we +want to index, so this works perfectly. The other change is that we add a `Annotated[..., UniqueItems]` wrapped around +the actual type. This is a Python construct that lets us add some metadata to a type annotation. The `UniqueItems` data +will instruct Algobattle (again using Pydantic) to also validate that the items in the wrapped collection are different +from each other. + +!!! tip "Annotated Metadata" + Using the `Annotated[...]` construct to add metadata is a powerful way to define validation, but can also be very + confusing to people new to the Python type system. We go over it in much more detail in the + [advanced types](annotations.md#advanced-type-annotations) section. + +We also need to check that the first two numbers actually have the same sum as the second two. This is best done with +a custom validation method: + +```py +class Solution(SolutionModel[Instance]): + """Solutions of Pairsum.""" + + indices: Annotated[tuple[SizeIndex, SizeIndex, SizeIndex, SizeIndex], UniqueItems] + + def validate_solution(self, instance: Instance, role: Role) -> None: + super().validate_solution(instance, role) + first = instance.numbers[self.indices[0]] + instance.numbers[self.indices[1]] + second = instance.numbers[self.indices[2]] + instance.numbers[self.indices[3]] + if first != second: + raise ValidationError("Solution elements don't have the same sum.") + + @maximize + def score(self, instance: Instance, role: Role) -> float: + ... +``` + +Note that this is now called `validate_solution` and takes not only the solution itself, but also the instance it is +trying to solve and the role of the team that created this solution as arguments. + +!!! note "Role Argument" + Most of the time the role argument won't be used to validate a solution. You must still have it listed in the + argument list of the method for everything to work smoothly. Some problems use this to e.g. relax some condition for + the solving team. + +### Solution Score + +Many problems not only care about a team providing a valid solution but also want them to compute the best solution they +can. For example, we might modify to not just want any four such numbers, but want the sum of each pair to be as big as +possible. For these problems we implement the `score` function. If we leave it out all solutions will be scored +equally. + +```py +class Solution(SolutionModel[Instance]): + """Solutions of Pairsum.""" + + indices: Annotated[tuple[SizeIndex, SizeIndex, SizeIndex, SizeIndex], UniqueItems] + + def validate_solution(self, instance: Instance, role: Role) -> None: + super().validate_solution(instance, role) + first = instance.numbers[self.indices[0]] + instance.numbers[self.indices[1]] + second = instance.numbers[self.indices[2]] + instance.numbers[self.indices[3]] + if first != second: + raise ValidationError("Solution elements don't have the same sum.") + + @maximize + def score(self, instance: Instance, role: Role) -> float: + return instance.numbers[self.indices[0]] + instance.numbers[self.indices[1]] +``` + +This method again receives the solution itself, the instance it solves, and the role of the team that generated it. It +needs to return a non-negative real number indicating how good the solution is. When using the `@maximize` decorator +(or using no decorator at all) bigger scores are considered better, if the problem instead asks for the smallest of +some value instead import and use the `@minimize` decorator. + +## Constructing the Problem + +Now that we have an Instance and a Solution class we can tie everything together using the Problem constructor. + +```py +Problem( + name="Pairsum", + min_size=4, + instance_cls=Instance, + solution_cls=Solution, +) +``` + +In its most basic form it just takes the name of the problem, and both classes we defined above. Finally, it also takes +a number that defines what the smallest reasonable instance size for this problem can be. This is needed because in our +case there aren't any sensible problem instances that only contain 3 numbers since we're looking for two pairs of two +numbers in the list. So if a generator was asked to create an instance of size 3 they couldn't possibly do this and +would fail immediately. To prevent bugs like that fill in `min_size` with whatever the smallest size your problem can +properly operate at is. + +In summary, our final Pairsum problem file looks like this + +```py title="problem.py" +from typing import Annotated + +from algobattle.problem import Problem, InstanceModel, SolutionModel +from algobattle.util import Role, ValidationError +from algobattle.types import u64, MinLen, SizeIndex, UniqueItems + + +class Instance(InstanceModel): + """An instance of a Pairsum problem.""" + + numbers: Annotated[list[u64], MinLen(4)] + + @property + def size(self) -> int: + return len(self.numbers) + + +class Solution(SolutionModel[Instance]): + """A solution to a Pairsum problem.""" + + indices: Annotated[tuple[SizeIndex, SizeIndex, SizeIndex, SizeIndex], UniqueItems] + + def validate_solution(self, instance: Instance, role: Role) -> None: + super().validate_solution(instance, role) + first = instance.numbers[self.indices[0]] + instance.numbers[self.indices[1]] + second = instance.numbers[self.indices[2]] + instance.numbers[self.indices[3]] + if first != second: + raise ValidationError("Solution elements don't have the same sum.") + + +Problem( + name="Pairsum", + min_size=4, + instance_cls=Instance, + solution_cls=Solution, +) +``` + +## Creating a Description + +Now that we've made the problem file to tell the framework how our problem works, we need to create a description file +to tell our students the same. This can be any file that just describes the problem in human-readable terms. It will be +packaged together with the problem file and distributed to the students. By default, Algobattle expects this file to be +named `description` with an appropriate file ending, e.g. `description.md`, `description.pdf`, etc. + +!!! tip "Web Framework" + When using the Algobattle web framework, Markdown files work best for this since they can be displayed inline on the + problem page. + +~~~md title="description.md" +# The Pairsum Problem + +The Pairsum problem asks you to find two pairs of numbers in a list that have the same sum. I.e.: + +**Given**: List `L = [z_1,...,z_n]` +**Question**: Are there pairwise different `a, b, c, d in [0,...,n-1]` such that `L[a] + L[b] = L[c] + L[d]`? + +I.e. given a list of natural numbers the task is to find two pairs of these numbers with the same sum. +The `size` of an instance limits the length of the list of numbers. + +The generator should create a hard to solve instance and a certificate solution to prove that such a pair of pairs +indeed exists. The generator should be able to efficiently find the solution for any input list. + +## Instances +An instance just contains the list of numbers. For example: +```json +{ + "numbers": [1, 2, 3, 4, 5] +} +``` + +## Solutions +A solution contains a list with the four indices `a, b, c, d` in this order. For example: +```json +{ + "indices": [1, 4, 2, 3] +} +``` +This is a valid solution since `L[1] + L[4] = 2 + 5 = 3 + 4 = L[2] + L[3]`. +~~~ + +## Packaging + +To easily distribute your problem to your students you can use the Algobattle CLI like this: + +```console +algobattle package problem +``` + +This creates a `pairsum.algo` file which contains all the info Algobattle needs to initialize a new project folder on +the student's computer with your problem. Note that it will then not only contain the problem file, but also the +description, and the Algobattle config file. + +!!! info "A peek behind the curtain" + This file really just is a zip file containing the mentioned files that have been preprocessed slightly. The file + extension is there to indicate that you shouldn't pack or unpack these files manually since the CLI tool expects + them to be formatted in a precise way. + +!!! tip "Web Framework" + This file is what the web framework expects you to upload, it will then be used to run matches on the server and be + distributed to the students. diff --git a/docs/teaching_concept/english.md b/docs/instructor/teaching_english.md similarity index 97% rename from docs/teaching_concept/english.md rename to docs/instructor/teaching_english.md index 853aaef8..ee62d07c 100644 --- a/docs/teaching_concept/english.md +++ b/docs/instructor/teaching_english.md @@ -1,7 +1,7 @@ -# Teaching Concept Lab Course "Algorithmic Battle" +# Lab Course Teaching Concept !!! info "This page also is available in German" - [Switch language :de:](german.md){ .md-button } + [Switch language :de:](teaching_german.md){ .md-button } ## Basics diff --git a/docs/teaching_concept/german.md b/docs/instructor/teaching_german.md similarity index 97% rename from docs/teaching_concept/german.md rename to docs/instructor/teaching_german.md index 5f62d73a..4b8b14e7 100644 --- a/docs/teaching_concept/german.md +++ b/docs/instructor/teaching_german.md @@ -1,7 +1,7 @@ -# Lehrkonzept Praktikum "Algorithmic Battle" +# Praktikum Lehrkonzept !!! info "Diese Seite ist auch auf Englisch verfügbar" - [Sprache wechseln :gb:](english.md){ .md-button } + [Sprache wechseln :gb:](teaching_english.md){ .md-button } ## Grundlegendes diff --git a/docs/src/problem/instance.py b/docs/src/problem/instance.py new file mode 100644 index 00000000..b5470c7f --- /dev/null +++ b/docs/src/problem/instance.py @@ -0,0 +1,12 @@ +Timespan = Annotated[int, Interval(ge=0, le=(2**64 - 1) / 5)] +Machine = Annotated[int, Interval(ge=1, le=5)] + + +class Instance(InstanceModel): + """The Scheduling problem class.""" + + job_lengths: list[Timespan] + + @property + def size(self) -> int: + return len(self.job_lengths) diff --git a/docs/src/problem/problem.py b/docs/src/problem/problem.py new file mode 100644 index 00000000..9060aa58 --- /dev/null +++ b/docs/src/problem/problem.py @@ -0,0 +1,41 @@ +"""The Scheduling problem class.""" +from typing import Annotated + +from algobattle.problem import Problem, InstanceModel, SolutionModel, minimize +from algobattle.types import Interval, SizeLen +from algobattle.util import Role + + +Timespan = Annotated[int, Interval(ge=0, le=(2**64 - 1) / 5)] +Machine = Annotated[int, Interval(ge=1, le=5)] + + +class Instance(InstanceModel): + """The Scheduling problem class.""" + + job_lengths: list[Timespan] + + @property + def size(self) -> int: + return len(self.job_lengths) + + +class Solution(SolutionModel[Instance]): + """A solution to a Job Shop Scheduling problem.""" + + assignments: Annotated[list[Machine], SizeLen] + + @minimize + def score(self, instance: Instance, role: Role) -> float: + finish_time = [0] * 5 + for duration, machine in zip(instance.job_lengths, self.assignments): + finish_time[machine - 1] += duration * machine + return max(finish_time) + + +Scheduling = Problem( + name="Job Shop Scheduling", + min_size=5, + instance_cls=Instance, + solution_cls=Solution, +) diff --git a/docs/src/problem/solution.py b/docs/src/problem/solution.py new file mode 100644 index 00000000..7a3353b8 --- /dev/null +++ b/docs/src/problem/solution.py @@ -0,0 +1,15 @@ +Timespan = Annotated[int, Interval(ge=0, le=(2**64 - 1) / 5)] +Machine = Annotated[int, Interval(ge=1, le=5)] + + +class Solution(SolutionModel[Instance]): + """A solution to a Job Shop Scheduling problem.""" + + assignments: Annotated[list[Machine], SizeLen] + + @minimize + def score(self, instance: Instance, role: Role) -> float: + finish_time = [0] * 5 + for duration, machine in zip(instance.job_lengths, self.assignments): + finish_time[machine - 1] += duration * machine + return max(finish_time) diff --git a/docs/tutorial/getting_started.md b/docs/tutorial/getting_started.md index 7393c4f3..167aef33 100644 --- a/docs/tutorial/getting_started.md +++ b/docs/tutorial/getting_started.md @@ -135,7 +135,7 @@ The filled in settings so far all just are paths to where Algobattle can find ce more things you can configure, but we're happy with the default values for now. /// tip -If you're curious what exactly everything in here means you can read the [config docs](/advanced/config.md). But for +If you're curious what exactly everything in here means you can read the [config docs](../advanced/config.md). But for now we recommend staying here since things will be much clearer after you're familiar with things here. /// diff --git a/docs/tutorial/index.md b/docs/tutorial/index.md index ab2dc96a..0e754ad4 100644 --- a/docs/tutorial/index.md +++ b/docs/tutorial/index.md @@ -1,4 +1,4 @@ -# Tutorial +# Student Tutorial The Algobattle tutorial goes over everything needed to understand how the package works and the most common ways to use it. It's aimed at students participating in a course and course instructors that want to learn how to work with the diff --git a/docs/tutorial/match.md b/docs/tutorial/match.md index b3bde1d9..857081c7 100644 --- a/docs/tutorial/match.md +++ b/docs/tutorial/match.md @@ -34,7 +34,7 @@ The last size where the solving team still was correct becomes that team's score !!! info "More details" The process described above is the best way to explain this battle type, but it's not actually precisely how it - works. You can find the actual process description in our [battle types](/advanced/battle_types.md) page. + works. You can find the actual process description in our [battle types](../advanced/battle_types.md) page. ## Let's get started @@ -57,7 +57,7 @@ little while. During this step Algobattle gets all the programs ready for execut Yes I can :wink:. The actual details of this are somewhat complicated if you're not familiar with Docker (and if you are, you'll have already figured our what's going on) so we recommend skipping over this for now. We recommend skipping over the details here for now and if you still want to learn more later you can check out the - [advanced guide on Docker](/advanced/docker.md#building-images). + [advanced guide on Docker](../advanced/docker.md#building-images). During this the interface will look something like this @@ -104,7 +104,7 @@ still running at the moment. ### Battle data On the right we see some data specific to the battle type. If you want to learn what the Iterated type displays here, -check out its documentation in the [battle types page](/advanced/battle_types.md#iterated). +check out its documentation in the [battle types page](../advanced/battle_types.md#iterated). ### Most recent fights diff --git a/docs/tutorial/summary.md b/docs/tutorial/summary.md index e75ef1e7..6af8b351 100644 --- a/docs/tutorial/summary.md +++ b/docs/tutorial/summary.md @@ -3,6 +3,6 @@ If you've made it all the way through the tutorial you're now probably ready to start coding things yourself! There's a lot of stuff left to cover, but none of it is essential to get started and can be more easily understood once you're a bit more familiar with the framework. Feel free to try things out and come back to either the tutorial, or the -[advanced topics](/advanced/index.md) whenever you're having questions. +[advanced topics](../advanced/index.md) whenever you're having questions. Hope you have fun with Algobattle! diff --git a/mkdocs.yml b/mkdocs.yml index 67f19abf..2ee33d38 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -66,8 +66,13 @@ markdown_extensions: - pymdownx.inlinehilite - pymdownx.details - pymdownx.superfences + - pymdownx.caret + - pymdownx.mark + - pymdownx.tilde - pymdownx.blocks.tab: alternate_style: true + - pymdownx.tabbed: + alternate_style: true - pymdownx.blocks.admonition: types: - note @@ -89,8 +94,8 @@ markdown_extensions: base_path: docs/src - attr_list - pymdownx.emoji: - emoji_index: !!python/name:materialx.emoji.twemoji - emoji_generator: !!python/name:materialx.emoji.to_svg + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg extra_css: @@ -115,10 +120,20 @@ nav: - advanced/config.md - advanced/battle_types.md - advanced/docker.md + - advanced/problems.md + - Instructor's Corner: + - instructor/index.md + - instructor/teaching_english.md + - Creating Problems: + - instructor/problem/intro.md + - instructor/problem/problem_file.md + - instructor/problem/example.md + - instructor/problem/advanced.md + - instructor/problem/annotations.md + - instructor/problem/io.md - API Reference: - api/index.md - api/battle.md - api/problem.md - api/util.md - - Teaching Concept: teaching_concept/english.md diff --git a/pyproject.toml b/pyproject.toml index a67d1b4e..01b45953 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,13 +19,13 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "docker~=6.1.3", - "pydantic~=2.4.0", - "anyio~=4.0.0", - "typer[all]~=0.9.0", - "typing-extensions~=4.8.0", - "tomlkit~=0.12.1", - "jinja2~=3.1.2", + "docker>=7.0.0", + "pydantic>=2.5.3", + "anyio>=4.2.0", + "typer[all]>=0.9.0", + "typing-extensions>=4.9.0", + "tomlkit>=0.12.3", + "jinja2>=3.1.2", ] [project.urls] @@ -35,14 +35,14 @@ Repository = "https://github.com/Benezivas/algobattle" [project.optional-dependencies] dev = [ - "black~=23.7.0", - "flake8~=6.0.0", - "flake8-docstrings~=1.7.0", - "mkdocs~=1.4.3", - "mkdocs-material~=9.1.18", - "pymdown-extensions~=10.0.1", - "mkdocstrings[python]~=0.22.0", - "mdx_include~=1.4.2", + "black>=23.12.1", + "flake8>=6.1.0", + "flake8-docstrings>=1.7.0", + "mkdocs>=1.5.3", + "mkdocs-material>=9.5.3", + "pymdown-extensions>=10.7", + "mkdocstrings[python]>=0.24.0", + "mdx-include>=1.4.2", ] [project.scripts]