From 826d992339c6d152ebc2fa74606d35de7e82dc04 Mon Sep 17 00:00:00 2001 From: Marlin Forbes Date: Sun, 14 Apr 2019 06:59:13 +0200 Subject: [PATCH] PHPCheck initial implementation --- .gitignore | 4 + .php_cs | 111 +++++++++ .travis.yml | 33 +++ README.md | 153 ++++++++++++ bin/phpcheck | 31 +++ checks/GenCheck.php | 196 ++++++++++++++++ composer.json | 28 ++- examples/EncoderCheck.php | 16 ++ examples/README.md | 9 + examples/encoder.php | 39 ++++ phpcheck.xml.dist | 11 + psalm.xml | 53 +++++ src/ArgumentFactory.php | 166 +++++++++++++ src/Check.php | 29 +++ src/CheckCommand.php | 35 +++ src/CheckEvents.php | 16 ++ src/Events/EndAllEvent.php | 9 + src/Events/EndEvent.php | 27 +++ src/Events/ErrorEvent.php | 10 + src/Events/Event.php | 18 ++ src/Events/FailureEvent.php | 10 + src/Events/ResultEvent.php | 42 ++++ src/Events/StartAllEvent.php | 9 + src/Events/StartEvent.php | 21 ++ src/Events/SuccessEvent.php | 10 + src/ExecutionError.php | 31 +++ src/ExecutionFailure.php | 31 +++ src/Gen.php | 335 +++++++++++++++++++++++++++ src/Reporters/ConsoleReporter.php | 160 +++++++++++++ src/Reporters/JUnitReporter.php | 67 ++++++ src/Reporters/Reporter.php | 42 ++++ src/RunState.php | 164 +++++++++++++ src/Runner.php | 371 ++++++++++++++++++++++++++++++ 33 files changed, 2277 insertions(+), 10 deletions(-) create mode 100644 .php_cs create mode 100644 .travis.yml create mode 100644 README.md create mode 100755 bin/phpcheck create mode 100644 checks/GenCheck.php create mode 100644 examples/EncoderCheck.php create mode 100644 examples/README.md create mode 100644 examples/encoder.php create mode 100644 phpcheck.xml.dist create mode 100644 psalm.xml create mode 100644 src/ArgumentFactory.php create mode 100644 src/Check.php create mode 100644 src/CheckCommand.php create mode 100644 src/CheckEvents.php create mode 100644 src/Events/EndAllEvent.php create mode 100644 src/Events/EndEvent.php create mode 100644 src/Events/ErrorEvent.php create mode 100644 src/Events/Event.php create mode 100644 src/Events/FailureEvent.php create mode 100644 src/Events/ResultEvent.php create mode 100644 src/Events/StartAllEvent.php create mode 100644 src/Events/StartEvent.php create mode 100644 src/Events/SuccessEvent.php create mode 100644 src/ExecutionError.php create mode 100644 src/ExecutionFailure.php create mode 100644 src/Gen.php create mode 100644 src/Reporters/ConsoleReporter.php create mode 100644 src/Reporters/JUnitReporter.php create mode 100644 src/Reporters/Reporter.php create mode 100644 src/RunState.php create mode 100644 src/Runner.php diff --git a/.gitignore b/.gitignore index c8153b5..5b794ce 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ /composer.lock +/.phpcheck/ +/php_errors.log +/.phpunit.result.cache /vendor/ +/.vscode/ diff --git a/.php_cs b/.php_cs new file mode 100644 index 0000000..cb39ef8 --- /dev/null +++ b/.php_cs @@ -0,0 +1,111 @@ +setRiskyAllowed(true) + ->setRules( + [ + '@PSR2' => true, + 'psr4' => true, + 'align_multiline_comment' => true, + 'array_syntax' => [ + 'syntax' => 'short', + ], + 'binary_operator_spaces' => [ + 'align_double_arrow' => false, + 'align_equals' => false, + ], + 'binary_operator_spaces' => true, + 'blank_line_after_opening_tag' => true, + 'blank_line_before_statement' => [ + 'statements' => [ + 'return', + 'throw', + ], + ], + 'cast_spaces' => [ + 'space' => 'single', + ], + 'concat_space' => [ + 'spacing' => 'one', + ], + 'declare_equal_normalize' => [ + 'space' => 'single', + ], + 'method_argument_space' => [ + 'keep_multiple_spaces_after_comma' => false, + 'on_multiline' => 'ensure_fully_multiline', + ], + 'method_chaining_indentation' => true, + 'function_typehint_space' => true, + 'general_phpdoc_annotation_remove' => [ + '@author', + '@package', + ], + 'include' => true, + 'indentation_type' => true, + 'line_ending' => true, + 'linebreak_after_opening_tag' => true, + 'lowercase_cast' => true, + 'method_separation' => true, + 'modernize_types_casting' => true, + 'native_function_casing' => true, + 'no_blank_lines_after_class_opening' => true, + 'no_blank_lines_after_phpdoc' => true, + 'no_closing_tag' => true, + 'no_empty_comment' => true, + 'no_empty_phpdoc' => true, + 'no_empty_statement' => true, + 'no_extra_consecutive_blank_lines' => true, + 'no_leading_import_slash' => true, + 'no_mixed_echo_print' => [ + 'use' => 'echo', + ], + 'no_multiline_whitespace_before_semicolons' => true, + 'no_php4_constructor' => true, + 'no_short_bool_cast' => true, + 'no_trailing_comma_in_singleline_array' => true, + 'no_trailing_whitespace' => true, + 'no_unused_imports' => true, + 'no_whitespace_before_comma_in_array' => true, + 'no_whitespace_in_blank_line' => true, + 'object_operator_without_whitespace' => true, + 'ordered_imports' => true, + 'phpdoc_add_missing_param_annotation' => [ + 'only_untyped' => false, + ], + 'phpdoc_indent' => true, + 'phpdoc_inline_tag' => true, + 'phpdoc_no_alias_tag' => [ + 'type' => 'var', + ], + 'phpdoc_no_empty_return' => true, + 'phpdoc_no_package' => true, + 'phpdoc_no_useless_inheritdoc' => true, + 'phpdoc_order' => true, + 'phpdoc_scalar' => true, + 'phpdoc_separation' => true, + 'phpdoc_trim' => true, + 'phpdoc_types' => true, + 'phpdoc_var_without_name' => true, + 'random_api_migration' => true, + 'return_type_declaration' => true, + 'short_scalar_cast' => true, + 'single_blank_line_before_namespace' => true, + 'single_line_comment_style' => [ + 'comment_types' => [ + 'asterisk', + 'hash', + ], + ], + 'single_quote' => true, + 'space_after_semicolon' => true, + 'standardize_not_equals' => true, + 'switch_case_semicolon_to_colon' => true, + 'ternary_operator_spaces' => true, + 'ternary_to_null_coalescing' => true, + 'trailing_comma_in_multiline_array' => true, + 'trim_array_spaces' => true, + 'unary_operator_spaces' => true, + 'yoda_style' => false, + ] + ); diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..6c1a3fa --- /dev/null +++ b/.travis.yml @@ -0,0 +1,33 @@ +language: php + +sudo: false + +php: + - 7.1 + - 7.2 + - 7.3 + +matrix: + fast_finish: true + +env: + matrix: + - DEPENDENCIES="high" + - DEPENDENCIES="low" + global: + - DEFAULT_COMPOSER_FLAGS="--no-interaction --no-ansi --no-progress --no-suggest" + +git: + depth: 3 + +install: + - if [[ "$DEPENDENCIES" = "high" ]]; then travis_retry composer update $DEFAULT_COMPOSER_FLAGS; fi + - if [[ "$DEPENDENCIES" = "low" ]]; then travis_retry composer update $DEFAULT_COMPOSER_FLAGS --prefer-lowest; fi + +script: + - bin/phpcheck + +cache: + directories: + - $HOME/.cache/composer/files + - $HOME/.composer/cache/files diff --git a/README.md b/README.md new file mode 100644 index 0000000..7d4f7fb --- /dev/null +++ b/README.md @@ -0,0 +1,153 @@ +# phpcheck + +![Development Status](https://img.shields.io/badge/status-alpha-red.svg) +[![Build Status](https://travis-ci.org/datashaman/phpcheck.svg?branch=master)](https://travis-ci.org/datashaman/phpcheck) + +PHP implementation of Haskell's QuickCheck. + +*NB* This is *ALPHA* status. Do not use it yet. + +Table of Contents +================= + + * [phpcheck](#phpcheck) + * [type declarations](#type-declarations) + * [annotations](#annotations) + * [generators](#generators) + * [examples](#examples) + * [command line arguments](#command-line-arguments) + * [storage of results](#storage-of-results) + +## type declarations + +`PHPCheck` will automatically generate arguments for check methods based on type declarations. For finer-grained control +over the arguments, use annotations on the method parameters. + +## annotations + +Annotate your check method parameters to control the arguments provided to the method. + +Parameter tags (use them in the description of a parameter, usually the end): + +* `{@gen name}` or `{@gen name:params}` where `name` is the name of the generator and `params` is a JSON encoded array of arguments passed to the generator. + +Method tags: + +* `@iterates` indicates that this check method handles its own iteration, and should be called once with no parameters. +* `@iterations` sets the number of iterations for this check method. The default is 100. + +## generators + +Below is the list of generators that are currently available: + +* `ascii(Generator $sizes = null)` +* `booleans(int $chanceOfGettingTrue = 50)` +* `characters($minChar, $maxChar)` +* `choose(array $arr)` +* `floats(float $min, float $max, Generator $decimals = null)` +* `integers(int $min = PHP_INT_MIN, int $max = PHP_INT_MAX)` +* `intervals(array $include = [[PHP_INT_MIN, PHP_INT_MAX]], array $exclude=[])` +* `listOf(Generator $values = null, Generator $sizes = null)` +* `strings(Generator $sizes = null, Generator $characters = null)` + +You have to nest the parameter tag to specify a generator argument. See (GenCheck.php)[checks/GenCheck.php] for examples. + +## examples + +There is an example check implemented in the _examples_ folder. To run it: + + phpcheck examples + +The [_Gen_ class checks](checks/GenCheck.php) for this package are a great illustration of the use of the generators. + +## command line arguments + +The `phpcheck` program accept a number of arguments and options: + + Description: + Runs checks. + + Usage: + phpcheck [options] [--] [] + + Arguments: + path File or folder with checks [default: "checks"] + + Options: + --bootstrap[=BOOTSTRAP] A PHP script that is included before the checks run + -f, --filter[=FILTER] Filter the checks that will be run + -i, --iterations=ITERATIONS How many times each check will be run [default: 100] + -j, --log-junit[=LOG-JUNIT] Log check execution in JUnit XML format to file + -d, --no-defects[=NO-DEFECTS] Ignore previous defects [default: false] + -h, --help Display this help message + -q, --quiet Do not output any message + -V, --version Display this application version + --ansi Force ANSI output + --no-ansi Disable ANSI output + -n, --no-interaction Do not ask any interactive question + -v|vv|vvv, --verbose Increase the verbosity of messages: 1 for normal output, 2 for more verbose output and 3 for debug + +The `--bootstrap` parameter can be included in a _phpcheck.xml_ or _phpcheck.xml.dist_ file. See [ours](phpcheck.xml.dist) for an example. + +The `--filter` or `-f` parameter is a filename-style match as follows: + + ClassName:: + ClassName::MethodName + MethodName + +where `ClassName` and `MethodName` can include patterns using `*` and `?` as you'd expect. + +The console reporter outputs check results much like `PHPUnit`: + + PHPCheck 0.1.0 by Marlin Forbes and contributors. + + ............. + + 13 / 13 (100%) + + Time: 284 ms, Memory: 6.00 MB + + OK (Checks: 13, Iterations: 120006, Failures: 0, Errors: 0) + +Using `---verbose 3` or `-vvv` enables a list of the checks as they are run: + + PHPCheck 0.1.0 by Marlin Forbes and contributors. + + Check 'Datashaman\PHPCheck\Checks\GenCheck::checkCharacters' started + Check 'Datashaman\PHPCheck\Checks\GenCheck::checkCharacters' ended + Check 'Datashaman\PHPCheck\Checks\GenCheck::checkStrings' started + Check 'Datashaman\PHPCheck\Checks\GenCheck::checkStrings' ended + Check 'Datashaman\PHPCheck\Checks\GenCheck::checkAscii' started + Check 'Datashaman\PHPCheck\Checks\GenCheck::checkAscii' ended + Check 'Datashaman\PHPCheck\Checks\GenCheck::checkBooleans' started + Check 'Datashaman\PHPCheck\Checks\GenCheck::checkBooleans' ended + Check 'Datashaman\PHPCheck\Checks\GenCheck::checkBooleansWithPercentage' started + Check 'Datashaman\PHPCheck\Checks\GenCheck::checkBooleansWithPercentage' ended + Check 'Datashaman\PHPCheck\Checks\GenCheck::checkCharactersWithNumbers' started + Check 'Datashaman\PHPCheck\Checks\GenCheck::checkCharactersWithNumbers' ended + Check 'Datashaman\PHPCheck\Checks\GenCheck::checkCharactersWithStrings' started + Check 'Datashaman\PHPCheck\Checks\GenCheck::checkCharactersWithStrings' ended + Check 'Datashaman\PHPCheck\Checks\GenCheck::checkChoose' started + Check 'Datashaman\PHPCheck\Checks\GenCheck::checkChoose' ended + Check 'Datashaman\PHPCheck\Checks\GenCheck::checkIterations' started + Check 'Datashaman\PHPCheck\Checks\GenCheck::checkIterations' ended + Check 'Datashaman\PHPCheck\Checks\GenCheck::checkFloats' started + Check 'Datashaman\PHPCheck\Checks\GenCheck::checkFloats' ended + Check 'Datashaman\PHPCheck\Checks\GenCheck::checkFloatsWithDecimalGen' started + Check 'Datashaman\PHPCheck\Checks\GenCheck::checkFloatsWithDecimalGen' ended + Check 'Datashaman\PHPCheck\Checks\GenCheck::checkStringsWithMinMax' started + Check 'Datashaman\PHPCheck\Checks\GenCheck::checkStringsWithMinMax' ended + Check 'Datashaman\PHPCheck\Checks\GenCheck::checkListOfInts' started + Check 'Datashaman\PHPCheck\Checks\GenCheck::checkListOfInts' ended + + Time: 305 ms, Memory: 6.00 MB + + OK (Checks: 13, Iterations: 120006, Failures: 0, Errors: 0) + +## storage of results + +`PHPCheck` stores results of check execution in the `.phpcheck` folder of the project. You should add that to your `.gitignore` file. + +When `PHPCheck` finds an error or failure, it will retry the defective arguments first before going onto regular iterations with new arguments. + +If you wish to ignore the previous defects and run through new iterations only, use `--no-defects` or `-d`. diff --git a/bin/phpcheck b/bin/phpcheck new file mode 100755 index 0000000..b757cf0 --- /dev/null +++ b/bin/phpcheck @@ -0,0 +1,31 @@ +#!/usr/bin/env php +add($command); +$application->setDefaultCommand($command->getName(), true); +$application->run(); diff --git a/checks/GenCheck.php b/checks/GenCheck.php new file mode 100644 index 0000000..fed831b --- /dev/null +++ b/checks/GenCheck.php @@ -0,0 +1,196 @@ += $interval[0] && $ord <= $interval[1]); + } + } + + public function checkStrings(string $string) + { + Assert::true(mb_strlen($string) <= 30); + } + + /** + * @param string $string {@gen ascii} + */ + public function checkAscii(string $string) + { + Assert::true(mb_strlen($string) <= 30); + + foreach ($this->mbSplit($string) as $character) { + Assert::range(ord($character), 0, 0x7F); + } + } + + /** + * @iterates + */ + public function checkBooleans() + { + $counts = [ + false => 0, + true => 0, + ]; + + $func = function (bool $b) use (&$counts) { + $counts[$b]++; + }; + + $this->runner->iterate( + $func, + 10000 + ); + + $total = $counts[false] + $counts[true]; + $percentageTrue = $counts[true] / $total * 100; + + Assert::true($percentageTrue >= 45 && $percentageTrue <= 55); + } + + /** + * @iterates + */ + public function checkBooleansWithPercentage() + { + $counts = [ + false => 0, + true => 0, + ]; + + /** + * @param bool $b {@gen booleans:[75]} + */ + $func = function (bool $b) use (&$counts) { + $counts[$b]++; + }; + + $this->runner->iterate( + $func, + 10000 + ); + + $total = $counts[false] + $counts[true]; + $percentageTrue = $counts[true] / $total * 100; + + Assert::true($percentageTrue >= 70 && $percentageTrue <= 80); + } + + /** + * @param string $c {@gen characters:[32,126]} + */ + public function checkCharactersWithNumbers(string $c) + { + $ord = mb_ord($c); + Assert::greaterThanEq($ord, 32); + Assert::lessThanEq($ord, 126); + } + + /** + * @param string $c {@gen characters:[" ","~"]} + */ + public function checkCharactersWithStrings(string $c) + { + $ord = mb_ord($c); + Assert::greaterThanEq($ord, 32); + Assert::lessThanEq($ord, 126); + } + + /** + * @param int $value {@gen choose:[[1,2,3]]} + */ + public function checkChoose(int $value) + { + Assert::range($value, 1, 3); + } + + /** + * @iterations 5 + */ + public function checkIterations() + { + static $iterations = 0; + $iterations++; + Assert::lessThanEq($iterations, 5); + } + + /** + * @param float $f {@gen floats:[0,5]} + */ + public function checkFloats(float $f) + { + Assert::greaterThanEq($f, 0); + Assert::lessThanEq($f, 5); + + if (preg_match('/\.([0-9]*)$/', (string) $f, $match)) { + Assert::lessThanEq(strlen($match[1]), 4); + } + } + + /** + * @param float $f {@gen floats:[0,5,{@gen integers:[4,4]}]} + */ + public function checkFloatsWithDecimalGen(float $f) + { + Assert::greaterThanEq($f, 0); + Assert::lessThanEq($f, 5); + + if (preg_match('/\.([0-9]*)$/', (string) $f, $match)) { + Assert::lessThanEq(strlen($match[1]), 4); + } + } + + /** + * @param string $s {@gen strings:[{@gen integers:[5,30]}]} + */ + public function checkStringsWithMinMax(string $s) + { + $count = mb_strlen($s); + Assert::lessThanEq($count, 30); + Assert::greaterThanEq($count, 5); + } + + /** + * @param array $list {@gen listOf:[{@gen integers:[0,10]},{@gen integers:[5,5]}]} + */ + public function checkListOfInts(array $list) + { + Assert::count($list, 5); + + foreach ($list as $value) { + Assert::integer($value); + } + } +} diff --git a/composer.json b/composer.json index 9097153..57395f1 100644 --- a/composer.json +++ b/composer.json @@ -1,6 +1,6 @@ { - "name": "datashaman/hypothesis", - "description": "PHP implementation of hypothesis testing.", + "name": "datashaman/phpcheck", + "description": "PHP implementation of Haskell's QuickCheck.", "type": "library", "keywords": [ "testing" @@ -15,15 +15,23 @@ "prefer-stable": true, "require": { "php": "^7.2", + "ext-json": "*", + "ext-libxml": "*", + "ext-mbstring": "*", + "ext-sqlite3": "*", + "ext-xml": "*", + "ext-xmlwriter": "*", + "fzaninotto/faker": "^1.8", + "nunomaduro/collision": "^3.0", + "phpdocumentor/reflection-docblock": "^4.3", "symfony/console": "^4.2", + "symfony/dependency-injection": "^4.2", + "symfony/event-dispatcher": "^4.2", "symfony/finder": "^4.2", - "symfony/property-info": "^4.2", - "symfony/stopwatch": "^4.2", - "phpdocumentor/reflection-docblock": "^4.3", - "phpunit/phpunit": "^8.0" + "symfony/var-dumper": "^4.2", + "webmozart/assert": "^1.4" }, "require-dev": { - "nunomaduro/collision": "^3.0" }, "config": { "platform": { @@ -33,16 +41,16 @@ "sort-packages": true }, "bin": [ - "hypothesis" + "bin/phpcheck" ], "autoload": { "psr-4": { - "Datashaman\\Hypothesis\\": "src/" + "Datashaman\\PHPCheck\\": "src/" } }, "autoload-dev": { "psr-4": { - "Datashaman\\Hypothesis\\Tests\\": "tests/" + "Datashaman\\PHPCheck\\Checks\\": "checks/" } } } diff --git a/examples/EncoderCheck.php b/examples/EncoderCheck.php new file mode 100644 index 0000000..ccd7630 --- /dev/null +++ b/examples/EncoderCheck.php @@ -0,0 +1,16 @@ + + + + + ./checks + + + + + + diff --git a/psalm.xml b/psalm.xml new file mode 100644 index 0000000..da8216f --- /dev/null +++ b/psalm.xml @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/ArgumentFactory.php b/src/ArgumentFactory.php new file mode 100644 index 0000000..5e4641b --- /dev/null +++ b/src/ArgumentFactory.php @@ -0,0 +1,166 @@ + 'booleans', + 'float' => 'floats', + 'int' => 'integers', + 'string' => 'strings', + ]; + + /** + * @var FakerGenerator + */ + protected $faker; + + /** + * @var Gen + */ + protected $gen; + + public function __construct(Gen $gen) + { + $this->faker = Factory::create(); + $this->gen = $gen; + } + + public function getParamAnnotations($reflectionCallable): array + { + $factory = DocBlockFactory::createInstance(); + $docComment = $reflectionCallable->getDocComment(); + + if ($docComment === false) { + return []; + } + + $docBlock = $factory->create($docComment); + + return $docBlock->getTagsByName('param'); + } + + protected function getParamAnnotation(ReflectionParameter $param): ?Param + { + $method = $param->getDeclaringFunction(); + $annotations = $this->getParamAnnotations($method); + + foreach ($annotations as $annotation) { + if ($annotation->getVariableName() === $param->getName()) { + return $annotation; + } + } + + return null; + } + + protected function getParamTags(ReflectionParameter $param): array + { + $annotation = $this->getParamAnnotation($param); + + if (!$annotation) { + return []; + } + + $tags = []; + + foreach ($annotation->getDescription()->getTags() as $tag) { + $tags[$tag->getName()] = (string) $tag->getDescription(); + } + + return $tags; + } + + protected function parseGenTag(string $tag) + { + $embeds = []; + + $tag = preg_replace_callback( + '/{@gen\s+([^\}]*)}/', + function ($matches) use (&$embeds) { + $id = uniqid('', true); + $embeds[$id] = $matches[1]; + + return json_encode($id); + }, + $tag + ); + + $parts = explode(':', $tag); + $generator = $parts[0]; + $args = []; + + if (count($parts) > 1) { + $args = array_map( + function ($arg) use ($embeds) { + if (is_string($arg) && array_key_exists($arg, $embeds)) { + $tag = $embeds[$arg]; + [$generator, $args] = $this->parseGenTag($tag); + + return $this->gen->$generator(...$args); + } + + return $arg; + }, + json_decode($parts[1], true) + ); + } + + return [$generator, $args]; + } + + public function make($function): Generator + { + $generators = []; + + foreach ($function->getParameters() as $param) { + $tags = $this->getParamTags($param); + + if (array_key_exists('gen', $tags)) { + [$generator, $args] = $this->parseGenTag($tags['gen']); + } else { + $paramType = $param->hasType() ? $param->getType() : null; + $type = $paramType ? $paramType->getName() : 'mixed'; + + if (!array_key_exists($type, self::TYPE_GENERATORS)) { + throw new Exception("No generator found for $type"); + } + + $generator = self::TYPE_GENERATORS[$type]; + $args = []; + } + + $generators[] = $this->gen->$generator(...$args); + } + + while (true) { + $arguments = []; + + foreach ($generators as $generator) { + while ($generator->valid()) { + $arguments[] = $generator->current(); + $generator->next(); + break; + } + } + + yield $arguments; + } + } +} diff --git a/src/Check.php b/src/Check.php new file mode 100644 index 0000000..85bab9c --- /dev/null +++ b/src/Check.php @@ -0,0 +1,29 @@ +runner = $runner; + $this->gen = $runner->getGen(); + } +} diff --git a/src/CheckCommand.php b/src/CheckCommand.php new file mode 100644 index 0000000..574c79a --- /dev/null +++ b/src/CheckCommand.php @@ -0,0 +1,35 @@ +setDescription('Runs checks.') + ->addOption('bootstrap', null, InputOption::VALUE_OPTIONAL, 'A PHP script that is included before the tests run') + ->addOption('filter', 'f', InputOption::VALUE_OPTIONAL, 'Filter the checks that will be run') + ->addOption('iterations', 'i', InputOption::VALUE_REQUIRED, 'How many times each check will be run', Runner::MAX_ITERATIONS) + ->addOption('log-junit', 'j', InputOption::VALUE_OPTIONAL, 'Log test execution in JUnit XML format to file') + ->addOption('no-defects', 'd', InputOption::VALUE_OPTIONAL, 'Ignore previous defects', false) + ->addArgument('path', InputArgument::OPTIONAL, 'File or folder with checks', 'checks'); + } + + protected function execute(InputInterface $input, OutputInterface $output) + { + (new Runner($this))->execute($input, $output); + } +} diff --git a/src/CheckEvents.php b/src/CheckEvents.php new file mode 100644 index 0000000..7695062 --- /dev/null +++ b/src/CheckEvents.php @@ -0,0 +1,16 @@ +method = $method; + $this->status = $status; + } +} diff --git a/src/Events/ErrorEvent.php b/src/Events/ErrorEvent.php new file mode 100644 index 0000000..fb6aeab --- /dev/null +++ b/src/Events/ErrorEvent.php @@ -0,0 +1,10 @@ +time = microtime(true); + } +} diff --git a/src/Events/FailureEvent.php b/src/Events/FailureEvent.php new file mode 100644 index 0000000..6ba7d1c --- /dev/null +++ b/src/Events/FailureEvent.php @@ -0,0 +1,10 @@ +method = $method; + $this->args = $args; + $this->cause = $cause; + } +} diff --git a/src/Events/StartAllEvent.php b/src/Events/StartAllEvent.php new file mode 100644 index 0000000..780dc4a --- /dev/null +++ b/src/Events/StartAllEvent.php @@ -0,0 +1,9 @@ +method = $method; + } +} diff --git a/src/Events/SuccessEvent.php b/src/Events/SuccessEvent.php new file mode 100644 index 0000000..2369c5e --- /dev/null +++ b/src/Events/SuccessEvent.php @@ -0,0 +1,10 @@ +getMessage() + ) + ); + + $this->args = $args; + $this->cause = $cause; + } +} diff --git a/src/ExecutionFailure.php b/src/ExecutionFailure.php new file mode 100644 index 0000000..38a0ed4 --- /dev/null +++ b/src/ExecutionFailure.php @@ -0,0 +1,31 @@ +getMessage() + ) + ); + + $this->args = $args; + $this->cause = $cause; + } +} diff --git a/src/Gen.php b/src/Gen.php new file mode 100644 index 0000000..cd30230 --- /dev/null +++ b/src/Gen.php @@ -0,0 +1,335 @@ +faker = Factory::create(); + $this->runner = $runner; + } + + /** + * @param int $chanceOfGettingTrue + */ + public function booleans(int $chanceOfGettingTrue = 50): Generator + { + Assert::natural($chanceOfGettingTrue); + Assert::lessThanEq($chanceOfGettingTrue, 100); + + while (true) { + yield mt_rand(1, 100) <= $chanceOfGettingTrue; + } + } + + /** + * @param float $min + * @param float $max + * @param Generator|null $decimals + * + * @return Generator + */ + public function floats( + float $min = PHP_FLOAT_MIN, + float $max = PHP_FLOAT_MAX, + Generator $decimals = null + ): Generator { + Assert::lessThanEq($min, $max); + + if (is_null($decimals)) { + $decimals = $this->integers(0, 9); + } + + $iteration = 0; + + /** + * @var int $decimal + */ + foreach ($decimals as $decimal) { + $f = max( + $min, + min( + round($min + mt_rand() / mt_getrandmax() * $iteration / ($this->runner->maxIterations - 1) * ($max - $min), $decimal), + $max + ) + ); + yield $f; + $iteration++; + } + } + + /** + * @param int $min + * @param int $max + * + * @return Generator + */ + public function integers( + int $min = PHP_INT_MIN, + int $max = PHP_INT_MAX + ): Generator { + Assert::lessThanEq($min, $max); + + $currentMax = $min; + + $iteration = 0; + + while (true) { + $currentMax = max( + $min, + min( + (int) ($iteration / ($this->runner->maxIterations - 1) * ($max - $min) + $min), + $max + ) + ); + yield mt_rand($min, $currentMax); + $iteration++; + } + } + + /** + * @param array $include + * @param array $exclude + * + * @return Generator + */ + public function intervals( + array $include = [[PHP_INT_MIN, PHP_INT_MAX]], + array $exclude = [] + ): Generator { + Assert::isList($include); + Assert::minCount($include, 1); + + $intervals = []; + + $max = 0; + + foreach ($include as $interval) { + Assert::isList($interval); + Assert::count($interval, 2); + Assert::allNatural($interval); + + [$start, $end] = $interval; + + Assert::lessThanEq($start, $end); + + $size = $end - $start; + + $intervals[] = [$max, $size - 1, $start]; + + $max += $size; + } + + $integers = $this->integers(0, $max - 1); + + foreach ($integers as $integer) { + foreach ($intervals as $interval) { + $start = $interval[0]; + + $end = $interval[1]; + + $value = $interval[2]; + + if ($integer >= $start && $integer <= $end) { + $value = $value + $integer - $start; + + foreach ($exclude as $excludeInterval) { + $start = $excludeInterval[0]; + $end = $excludeInterval[1]; + + if ($value >= $start && $value <= $end) { + break 2; + } + } + + yield $value; + break; + } + } + } + } + + /** + * @param string|int|null $minChar A character, or its codepoint or ordinal value. + * @param string|int|null $maxChar A character, or its codepoint or ordinal value. + * + * @return Generator + */ + public function characters( + $minChar = null, + $maxChar = null + ): Generator { + if (is_null($minChar)) { + $minCodepoint = self::MIN_UNICODE; + } elseif (is_string($minChar)) { + $minCodepoint = $minChar === '' ? self::MIN_UNICODE : mb_ord($minChar); + } else { + Assert::integer($minChar); + $minCodepoint = $minChar; + } + + if (is_null($maxChar)) { + $maxCodepoint = self::MAX_UNICODE; + } elseif (is_string($maxChar)) { + $maxCodepoint = mb_ord($maxChar); + } else { + Assert::integer($minChar); + $maxCodepoint = $maxChar; + } + + Assert::lessThanEq($minCodepoint, $maxCodepoint); + Assert::greaterThanEq($minCodepoint, self::MIN_UNICODE); + Assert::lessThanEq($maxCodepoint, self::MAX_UNICODE); + + $codepoints = $this->intervals( + [ + [$minCodepoint, $maxCodepoint] + ], + self::EXCLUDE_UNICODE + ); + + foreach ($codepoints as $codepoint) { + yield mb_chr($codepoint); + } + } + + /** + * @param Generator|null $sizes + * @param Generator|null $characters + * + * @return Generator + */ + public function strings( + Generator $sizes = null, + Generator $characters = null + ): Generator { + if (is_null($sizes)) { + $sizes = $this->integers(0, self::DEFAULT_SIZE); + } + + if (is_null($characters)) { + $characters = $this->characters(); + } + + foreach ($sizes as $size) { + $result = ''; + + if ($size === 0) { + yield $result; + } + + while ($characters->valid()) { + $result .= (string) $characters->current(); + + if (mb_strlen($result) >= $size) { + yield $result; + break; + } + + $characters->next(); + } + } + } + + /** + * @param Generator|null $sizes + * + * @return Generator + */ + public function ascii( + Generator $sizes = null + ): Generator { + if (is_null($sizes)) { + $sizes = $this->integers(0, self::DEFAULT_SIZE); + } + + $strings = $this->strings( + $sizes, + $this->characters(0, 0x7F) + ); + + foreach ($strings as $string) { + yield $string; + } + } + + /** + * @param array $arr + * + * @return Generator + */ + public function choose( + array $arr + ): Generator { + Assert::notEmpty($arr); + Assert::isList($arr); + + $positions = $this->integers(0, count($arr) - 1); + + foreach ($positions as $position) { + yield $arr[$position]; + } + } + + /** + * @param Generator $values + * @param Generator $sizes + * + * @return Generator + */ + public function listOf( + Generator $values, + Generator $sizes = null + ): Generator { + if (is_null($sizes)) { + $sizes = $this->integers(0, self::DEFAULT_SIZE); + } + + foreach ($sizes as $size) { + $result = []; + + while ($values->valid()) { + $result[] = $values->current(); + + if (count($result) >= $size) { + yield $result; + break; + } + + $values->next(); + } + } + } +} diff --git a/src/Reporters/ConsoleReporter.php b/src/Reporters/ConsoleReporter.php new file mode 100644 index 0000000..d860779 --- /dev/null +++ b/src/Reporters/ConsoleReporter.php @@ -0,0 +1,160 @@ + 'F', + 'ERROR' => 'E', + 'SUCCESS' => '.', + ]; + + const STATUS_FORMATS = [ + 'FAILURE' => 'error', + 'ERROR' => 'error', + 'SUCCESS' => 'info', + ]; + + protected $writer; + + public function __construct(Runner $runner) + { + parent::__construct($runner); + + $baseDir = realpath(__DIR__ . '/../'); + + $this->writer = new Writer(); + $this->writer->ignoreFilesIn( + [ + '#' . $baseDir . '/src/*#', + '#' . $baseDir . '/bin/*#', + '#vendor/symfony/console.*#', + '#vendor/webmozart/assert.*#', + ] + ); + } + + public static function getSubscribedEvents(): array + { + return [ + CheckEvents::END_ALL => 'onEndAll', + CheckEvents::FAILURE => 'onFailure', + CheckEvents::END => 'onEnd', + CheckEvents::ERROR => 'onError', + CheckEvents::START => 'onStart', + CheckEvents::START_ALL => 'onStartAll', + ]; + } + + public function onStartAll(Events\StartAllEvent $event): void + { + $this->output->writeln( + sprintf(self::HEADER, CheckCommand::VERSION) + ); + $this->output->writeln(''); + } + + public function onStart(Events\StartEvent $event): void + { + if ($this->output->isDebug()) { + $signature = $this->getMethodSignature($event->method); + $this->output->writeln("Check '$signature' started"); + } + } + + public function onEnd(Events\EndEvent $event): void + { + if ($this->output->isDebug()) { + $signature = $this->getMethodSignature($event->method); + $this->output->writeln("Check '$signature' ended"); + } else { + $char = self::STATUS_CHARACTERS[$event->status]; + $format = self::STATUS_FORMATS[$event->status]; + $this->output->write("<$format>$char"); + } + } + + public function onEndAll(Events\EndAllEvent $event): void + { + $successes = count($this->state->successes); + $total = $successes + count($this->state->failures); + + if (!$this->output->isDebug()) { + $percentage = $total ? (int) ($successes / $total * 100) : 0; + + $this->output->writeln(''); + $this->output->writeln(''); + $this->output->writeln("$successes / $total ($percentage%)"); + } + + $seconds = $event->time - $this->state->startTime; + + if ($seconds < 1) { + $time = (int) ($seconds * 1000) . ' ms'; + } else { + $time = round($seconds, 2) . ' seconds'; + } + + $memory = $this->convertBytes(memory_get_peak_usage(true)); + + $this->output->writeln(''); + $this->output->writeln("Time: $time, Memory: $memory"); + + if ($failures = count($this->state->failures)) { + $this->writer->setOutput($this->output); + + $message = $failures === 1 ? 'There was 1 failure:' : "There were $failures failures:"; + + $this->output->writeln(''); + $this->output->writeln($message); + $this->output->writeln(''); + + foreach ($this->state->failures as $index => $failure) { + $number = $index + 1; + $signature = $this->getMethodSignature($failure->method); + $this->output->writeln("$number) $signature"); + + $inspector = new Inspector($failure->cause); + $this->writer->write($inspector); + $this->output->writeln(''); + } + } + + if ($errors = count($this->state->errors)) { + $this->writer->setOutput($this->output); + + $message = $errors === 1 ? 'There was 1 error:' : "There were $errors errors:"; + + $this->output->writeln(''); + $this->output->writeln($message); + $this->output->writeln(''); + + foreach ($this->state->errors as $index => $error) { + $number = $index + 1; + $signature = $this->getMethodSignature($error->method); + $this->output->writeln("$number) $signature"); + + $inspector = new Inspector($error->cause); + $this->writer->write($inspector); + $this->output->writeln(''); + } + } + + $this->output->writeln(''); + + $stats = "(Checks: $total, Iterations: {$this->runner->getTotalIterations()}, Failures: $failures, Errors: $errors)"; + $this->output->writeln($failures ? "FAILURES $stats" : "OK $stats"); + } +} diff --git a/src/Reporters/JUnitReporter.php b/src/Reporters/JUnitReporter.php new file mode 100644 index 0000000..341f308 --- /dev/null +++ b/src/Reporters/JUnitReporter.php @@ -0,0 +1,67 @@ + 'onEndAll', + CheckEvents::FAILURE => 'onFailure', + CheckEvents::ERROR => 'onError', + CheckEvents::START => 'onStart', + CheckEvents::START_ALL => 'onStartAll', + ]; + } + + public function onStartAll() + { + $this->testsuite = new SimpleXMLElement(''); + $this->testcase = null; + } + + public function onStart(Events\StartEvent $event) + { + $this->testcase = $this->testsuite->addChild('testcase'); + $this->testcase['classname'] = $event->method->getDeclaringClass()->getName(); + $this->testcase['name'] = $event->method->getName(); + } + + public function onError(Events\ErrorEvent $event) + { + $error = $this->testcase->addChild('error'); + $error['type'] = get_class($event->cause); + $error['message'] = sprintf( + "args=%s caused error '%s'", + json_encode($event->args), + $event->cause->getMessage() + ); + } + + public function onFailure(Events\FailureEvent $event) + { + $failure = $this->testcase->addChild('failure'); + $failure['type'] = get_class($event->cause); + $failure['message'] = sprintf( + "args=%s caused failure '%s'", + json_encode($event->args), + $event->cause->getMessage() + ); + } + + public function onEndAll(Events\EndAllEvent $event) + { + $this->testsuite->asXML($this->input->getOption('log-junit')); + } +} diff --git a/src/Reporters/Reporter.php b/src/Reporters/Reporter.php new file mode 100644 index 0000000..3274c80 --- /dev/null +++ b/src/Reporters/Reporter.php @@ -0,0 +1,42 @@ +input = $runner->getInput(); + $this->output = $runner->getOutput(); + $this->runner = $runner; + $this->state = $runner->getState(); + } + + public function getMethodSignature(ReflectionMethod $method): string + { + return $method->getDeclaringClass()->getName() . '::' . $method->getName(); + } + + protected function convertBytes(int $bytes): string + { + if ($bytes == 0) { + return "0.00 B"; + } + + $s = array('B', 'KB', 'MB', 'GB', 'TB', 'PB'); + $e = (int) floor(log($bytes, 1024)); + + return sprintf('%.2f %s', round($bytes / pow(1024, $e), 2), $s[$e]); + } +} diff --git a/src/RunState.php b/src/RunState.php new file mode 100644 index 0000000..ca5ce73 --- /dev/null +++ b/src/RunState.php @@ -0,0 +1,164 @@ + 'onFailure', + CheckEvents::ERROR => 'onError', + CheckEvents::START_ALL => 'onStartAll', + CheckEvents::SUCCESS => 'onSuccess', + ]; + } + + public function onError(Events\FailureEvent $event): void + { + $this->errors[] = $event; + $this->saveResult($event); + } + + public function onFailure(Events\FailureEvent $event): void + { + $this->failures[] = $event; + $this->saveResult($event); + } + + public function onStartAll(Events\StartAllEvent $event): void + { + $this->errors = []; + $this->failures = []; + $this->successes = []; + $this->startTime = $event->time; + } + + public function onSuccess(Events\SuccessEvent $event): void + { + $this->successes[] = $event; + $this->saveResult($event); + } + + protected function saveResult(Events\ResultEvent $event) + { + $args = $event->args ? (json_encode($event->args) ?: '') : ''; + $sql = sprintf( + self::INSERT_RESULT_SQL, + SQLite3::escapeString($event->method->getDeclaringClass()->getName()), + SQLite3::escapeString($event->method->getName()), + SQLite3::escapeString($event->status), + SQLite3::escapeString($this->formatMicrotime($event->time)), + SQLite3::escapeString($args) + ); + + $db = $this->getResultsDatabase(); + $db->exec($sql); + } + + protected function formatMicrotime(float $microtime): string + { + if (preg_match('/^[0-9]*\\.([0-9]+)$/', (string) $microtime, $reg)) { + $decimal = substr(str_pad($reg[1], 6, "0"), 0, 6); + } else { + $decimal = "000000"; + } + + $format = preg_replace('/(%f)/', $decimal, '%F %T.%f'); + + return strftime($format, (int) $microtime); + } + + protected function getResultsDatabase(): SQLite3 + { + /** + * @var SQLite3 $db + */ + static $db; + + if (!isset($db)) { + if (!is_dir(self::DATABASE_DIR)) { + mkdir(self::DATABASE_DIR, 0755); + } + + $db = new SQLite3(self::DATABASE_DIR . DIRECTORY_SEPARATOR . 'results.db'); + $db->exec(self::CREATE_RESULTS_SQL); + } + + return $db; + } + + public function getDefectArgs(ReflectionMethod $method): ?array + { + $sql = sprintf( + self::SELECT_DEFECT_SQL, + SQLite3::escapeString($method->getDeclaringClass()->getName()), + SQLite3::escapeString($method->getName()) + ); + + $db = $this->getResultsDatabase(); + $result = $db->query($sql); + + /** + * @var array $failure + */ + if (!$result) { + return null; + } + + $failure = $result->fetchArray(); + + if ($failure) { + /** + * @var string $args + */ + $args = $failure['args']; + + return (array) json_decode($args, true); + } + + return null; + } +} diff --git a/src/Runner.php b/src/Runner.php new file mode 100644 index 0000000..45068fb --- /dev/null +++ b/src/Runner.php @@ -0,0 +1,371 @@ +argumentFactory = new ArgumentFactory($gen); + $this->command = $command; + $this->dispatcher = new EventDispatcher(); + $this->gen = $gen; + $this->state = new RunState(); + + $this->dispatcher->addSubscriber($this->state); + } + + public static function getSubscribedEvents(): array + { + return [ + CheckEvents::END_ALL => 'onEndAll', + CheckEvents::FAILURE => 'onFailure', + CheckEvents::END => 'onEnd', + CheckEvents::ERROR => 'onError', + CheckEvents::START => 'onStart', + CheckEvents::START_ALL => 'onStartAll', + CheckEvents::SUCCESS => 'onSuccess', + ]; + } + + public function getGen() + { + return $this->gen; + } + + public function getInput() + { + return $this->input; + } + + public function getOutput() + { + return $this->output; + } + + public function getState() + { + return $this->state; + } + + public function getTotalIterations() + { + return $this->totalIterations; + } + + public function iterate( + callable $callable, + int $iterations = null + ): void { + $function = new ReflectionFunction($callable); + $arguments = $this->argumentFactory->make($function); + + $maxIterations = is_null($iterations) + ? $this->maxIterations + : $iterations; + + $iterations = 0; + + while ($arguments->valid()) { + $args = $arguments->current(); + + try { + call_user_func($callable, ...$args); + $iterations++; + } + + catch (InvalidArgumentException $exception) { + $this->totalIterations += $iterations + 1; + throw new ExecutionFailure($args, $exception); + } + + catch (Throwable $throwable) { + $this->totalIterations += $iterations + 1; + throw new ExecutionError($args, $throwable); + } + + if ($iterations++ >= $maxIterations - 1) { + break; + } + + $arguments->next(); + } + + $this->totalIterations += $iterations; + } + + public function execute(InputInterface $input, OutputInterface $output) + { + $this->input = $input; + $this->output = $output; + + $config = $this->getConfig(); + + $this->maxIterations = (int) $input->getOption('iterations'); + + $pathArgument = $input->getArgument('path'); + $path = realpath($pathArgument); + + if ($pathArgument && !$path) { + $output->writeln('Path does not exist'); + exit(1); + } + + if ($path && !file_exists($path)) { + $output->writeln('Path does not exist'); + exit(1); + } + + $this->dispatcher->addSubscriber(new Reporters\ConsoleReporter($this)); + + if ($input->getOption('log-junit')) { + $reporter = new Reporters\JUnitReporter($this); + $this->dispatcher->addSubscriber($reporter); + } + + $bootstrap = $input->getOption('bootstrap') + ?: $config['bootstrap'] + ?: null; + + if ($bootstrap) { + include_once $bootstrap; + } + + $event = new Events\StartAllEvent(); + $this->dispatcher->dispatch(CheckEvents::START_ALL, $event); + + $paths = $this->gatherPaths($path); + + [$classFilter, $methodFilter] = $this->getFilter($input); + + foreach ($paths as $path) { + $classes = get_declared_classes(); + + include $path; + + $classes = array_diff( + get_declared_classes(), + $classes, + [ + Check::class, + ] + ); + + $classes = array_filter( + $classes, + function ($class) { + return preg_match('/Check$/', $class) === 1; + } + ); + + $testClass = array_pop($classes); + + if ($classFilter && !fnmatch($classFilter, $testClass)) { + continue; + } + + $test = new $testClass($this); + + $class = new ReflectionClass($test); + + foreach ($class->getMethods() as $method) { + $name = $method->getName(); + + if ($methodFilter && !fnmatch($methodFilter, $name)) { + continue; + } + + if (preg_match('/^check/', $method->getName()) !== 1) { + continue; + } + + $tags = $this->getTags($method); + + // If method has an @iterates tag, + // it handles its own iteration internally + // and should be called normally with no args. + $iterates = false; + + $iterations = $this->maxIterations; + + foreach ($tags as $tag) { + $tagName = $tag->getName(); + + if ($tagName === 'iterates') { + $iterates = true; + break; + } + + if ($tagName === 'iterations') { + $iterations = (int) (string) $tag->getDescription(); + break; + } + } + + $closure = $method->getClosure($test); + + $event = new Events\StartEvent($method); + $this->dispatcher->dispatch(CheckEvents::START, $event); + + $defectArgs = $this->state->getDefectArgs($method); + + try { + if ($iterates) { + call_user_func($closure); + } else { + $noDefects = ($input->getOption('no-defects') !== false); + + if (!$noDefects && $defectArgs) { + try { + call_user_func($closure, ...$defectArgs); + } + + catch (InvalidArgumentException $exception) { + throw new ExecutionFailure($defectArgs, $exception); + } + + catch (Throwable $throwable) { + throw new ExecutionError($defectArgs, $throwable); + } + } + + $this->iterate($closure, $iterations); + } + + $event = new Events\SuccessEvent($method); + $this->dispatcher->dispatch(CheckEvents::SUCCESS, $event); + $status = 'SUCCESS'; + } + + catch (ExecutionFailure $failure) { + $event = new Events\FailureEvent( + $method, + $failure->args, + $failure->cause + ); + $this->dispatcher->dispatch(CheckEvents::FAILURE, $event); + $status = 'FAILURE'; + } + + catch (ExecutionError $error) { + $event = new Events\ErrorEvent( + $method, + $error->args, + $error->cause + ); + $this->dispatcher->dispatch(CheckEvents::ERROR, $event); + $status = 'ERROR'; + } + + $event = new Events\EndEvent($method, $status); + $this->dispatcher->dispatch(CheckEvents::END, $event); + } + } + + $event = new Events\EndAllEvent(); + $this->dispatcher->dispatch(CheckEvents::END_ALL, $event); + } + + protected function getConfig(string $filename = null): ?SimpleXMLElement + { + $filename = self::CONFIG_FILE; + + $filenames = [ + $filename, + "$filename.dist", + ]; + + foreach ($filenames as $filename) { + if (file_exists($filename)) { + return simplexml_load_file($filename) ?: null; + } + } + + return null; + } + + protected function gatherPaths($path): array + { + $paths = []; + + if (is_file($path)) { + $paths[] = $path; + } else { + $finder = new Finder(); + $finder->files()->in($path)->name('*Check.php'); + + if ($finder->hasResults()) { + foreach ($finder as $file) { + $paths[] = $file->getRealPath(); + } + } + } + + return $paths; + } + + protected function getTags(ReflectionMethod $method) + { + $factory = DocBlockFactory::createInstance(); + $docComment = $method->getDocComment(); + + if ($docComment === false) { + return []; + } + + $docBlock = $factory->create($docComment); + + return $docBlock->getTags(); + } + + protected function getFilter(InputInterface $input) + { + $filter = $input->getOption('filter'); + + if (!$filter) { + return [null, null]; + } + + $parts = explode('::', $filter); + + Assert::countBetween($parts, 1, 2); + + if (count($parts) === 1) { + return [null, $parts[0]]; + } + + return $parts; + } +}