Merge pull request #133 from hartwork/fix-ci

Make UI testing significantly more robust

.github/workflows/linux_and_macos.yml

@@ -81,12 +81,9 @@ jobs:
         run: |-
           brew tap homebrew/cask-fonts
           brew install \
-            agg \
             bmake \
             bsdmake \
-            coreutils \
-            font-liberation \
-            imagemagick
+            coreutils
 
       - name: Install build dependency Clang ${{ matrix.clang_major_version }}
         if: "${{ runner.os == 'Linux' && contains(matrix.cxx, 'clang') }}"
@@ -146,50 +143,25 @@ jobs:
           [[ "$(find ROOT/ -not -type d | tee /dev/stderr)" == '' ]]  # i.e. fail CI if leftover files
 
       - name: 'Run UI tests'
-        if: "${{ runner.os == 'macOS' }}"
         run: |-
           ./recordings/record.sh
           rm -Rf recordings/venv/
 
       - name: 'Upload UI test renderings for inspection'
-        if: "${{ runner.os == 'macOS' }}"
         uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32  # v3.1.3
         with:
-          name: ttyplot_ui_test_${{ github.sha }}_${{ matrix.runs-on }}_${{ matrix.cc }}
+          name: ttyplot_ui_test_${{ github.sha }}_${{ matrix.runs-on }}_${{ matrix.cc }}_${{ matrix.make }}
           path: recordings/actual*
           if-no-files-found: error
 
       - name: 'Evaluate UI test results'
-        if: "${{ runner.os == 'macOS' }}"
         run: |-
-          assert_images_equal_enough() {
-            local a="${1}"
-            local b="${2}"
-            local diff_output="${3}"
-            local dissimilarity="$(compare -metric DSSIM "${a}" "${b}" "${diff_output}" 2>&1)"
-            if ! python3 <<<"import sys; sys.exit(int(${dissimilarity} > 0.0022))"; then
-              echo "Image \"${a}\" is not close enough of a match to image \"${b}\", dissimilarity is ${dissimilarity}." >&2
-              return 1
-            fi
-            true
-          }
-
-          cd recordings/
-
-          error=0
-
-          for expected in expected*.png; do
-            actual=${expected/expected/actual}
-            diff=${expected/expected/diff}
-
-            assert_images_equal_enough ${actual} ${expected} ${diff} || error=1
-          done
-
-          rm -f actual*.* diff*.*
+          diff -u recordings/{expected,actual}.txt
+          rm -f recordings/actual.txt
 
           cat <<"EOF"
             ################################################################
-            ## If this step FAILS you can get the expected images
+            ## If this step FAILS you can get the expected ANSI screenshots
             ## back in sync with reality by running:
             ##
             ##   $ ./recordings/get_back_in_sync.sh

CONTRIBUTING.md

@@ -6,51 +6,38 @@ never be complete, but if something important
 turns out to be missing here, let's add it!
 
 
-## Problem: The CI complains that two images are too dissimilar
+## Problem: The CI complains that the UI test failed
 
-The CI does basic UI testing by making screenshots
+The CI does basic UI testing by making ANSI "screenshots"
 of the running application and comparing them
-against pre-recorded screenshots `recordings/expected-*.png`.
+against pre-recorded ANSI "screenshots" `recordings/expected.txt`.
 When you make changes to ttyplot that change the *runtime appearance* of
 ttyplot, the CI will hopefully catch these changes
 and reject them as a regression.  In a case where these
-changes are made with full intention, the images
-that the CI is comparing to at `recordings/expected-*.png`
+changes are made with full intention, the screenshots
+that the CI is comparing to at `recordings/expected.txt`
 will then need to be regenerated.
 For your convenience, an easy way to do that is:
 
 ```console
 $ ./recordings/get_back_in_sync.sh
 ```
 
-The script will re-render these images and even create a Git commit
+The script will re-render these screenshots and even create a Git commit
 for you.
 
 For all that to work well on macOS, you would need to install:
 
 ```console
 $ brew tap homebrew/cask-fonts
 $ brew install \
-    agg \
-    asciinema \
-    coreutils \
-    font-liberation
+    coreutils
 ```
 
-On a Debian-based Linux including Ubuntu, you would need to install…
+On a Debian-based Linux including Ubuntu, you would need to install:
 
 ```console
 $ sudo apt-get update
 $ sudo apt-get install --no-install-recommends -V \
-    ca-certificates \
-    cargo \
-    fonts-liberation \
     python3-venv
-$ cargo install --git https://github.com/asciinema/agg
-```
-
-…and put the install location of `agg` into `${PATH}`:
-
-```console
-$ export PATH="${PATH}:${HOME}/.cargo/bin"
 ```

recordings/.gitignore

@@ -1,3 +1,2 @@
 /venv/
-/actual*.*
-/diff*.png
+/actual.txt

recordings/expected.txt

@@ -0,0 +1,72 @@
+[90x20] Frame 1:
++------------------------------------------------------------------------------------------+
+|                                       .: ttyplot :.                                      |
+|  ^ 0.0                                                                                   |
+|  │                                                                                       |
+|  │                                                                                       |
+|  │                                                                                       |
+|  │ 0.0                                                                                   |
+|  │                                                                                       |
+|  │                                                                                       |
+|  │                                                                                       |
+|  │ 0.0                                                                                   |
+|  │                             waiting for data from stdin                               |
+|  │                                                                                       |
+|  │                                                                                       |
+|  │ 0.0                                                                                   |
+|  │                                                                                       |
+|  │                                                                                       |
+|  │                                                                                       |
+|  └─────────────────────────────────────────────────────────────────────────────────────> |
+|     X                                                           Thu Jan  1 00:00:00 1970 |
+|                                                  https://github.com/tenox7/ttyplot 1.6.0 |
++------------------------------------------------------------------------------------------+
+
+[90x20] Frame 2:
++------------------------------------------------------------------------------------------+
+|                                       .: ttyplot :.                                      |
+|  ^ 4.0                                                                                   |
+|  │                                                                                       |
+|  │                                                                                       |
+|  │                                                                                       |
+|  │ 3.0                                                                                 X |
+|  │                                                                                     X |
+|  │                                                                                     X |
+|  │                                                                                     X |
+|  │ 2.0                                                                                 X |
+|  │                                                                                     X |
+|  │                                                                                     X |
+|  │                                                                                     X |
+|  │ 1.0                                                                                XX |
+|  │                                                                                    XX |
+|  │                                                                                    XX |
+|  │                                                                                    XX |
+|  └─────────────────────────────────────────────────────────────────────────────────────> |
+|     X last=3.0 min=1.0 max=3.0 avg=2.0                          Thu Jan  1 00:00:00 1970 |
+|       last=4.0 min=2.0 max=4.0 avg=3.0           https://github.com/tenox7/ttyplot 1.6.0 |
++------------------------------------------------------------------------------------------+
+
+[90x20] Frame 3:
++------------------------------------------------------------------------------------------+
+|                                       .: ttyplot :.                                      |
+|  ^ 4.0                                                                                   |
+|  │                                                                                       |
+|  │                                                                                       |
+|  │                                                                                       |
+|  │ 3.0                                                                                 X |
+|  │                                                                                     X |
+|  │                                                                                     X |
+|  │                                                                                     X |
+|  │ 2.0                                                                                 X |
+|  │                                 input stream closed                                 X |
+|  │                                                                                     X |
+|  │                                                                                     X |
+|  │ 1.0                                                                                XX |
+|  │                                                                                    XX |
+|  │                                                                                    XX |
+|  │                                                                                    XX |
+|  └─────────────────────────────────────────────────────────────────────────────────────> |
+|     X last=3.0 min=1.0 max=3.0 avg=2.0                          Thu Jan  1 00:00:00 1970 |
+|       last=4.0 min=2.0 max=4.0 avg=3.0           https://github.com/tenox7/ttyplot 1.6.0 |
++------------------------------------------------------------------------------------------+
+

recordings/flip_book.py

@@ -0,0 +1,167 @@
+#! /usr/bin/env python3
+# Copyright (c) 2015 by pyte authors and contributors
+# Copyright (c) 2023 by Sebastian Pipping <sebastian@pipping.org>
+#
+# Licensed under LGPL v3, see pyte's LICENSE file for more details.
+#
+# Based on pyte's example "nanoterm.py"
+# https://raw.githubusercontent.com/selectel/pyte/master/examples/nanoterm.py
+# and a few lines from
+# https://github.com/selectel/pyte/blob/master/pyte/screens.py
+
+import enum
+import os
+import pty
+import select
+import signal
+import sys
+import time
+from functools import lru_cache
+from typing import Callable, Generator
+
+import pyte
+from pyte.screens import Char, StaticDefaultDict
+from wcwidth import wcwidth as _wcwidth  # type: ignore[import-untyped]
+
+
+wcwidth: Callable[[str], int] = lru_cache(maxsize=4096)(_wcwidth)  # from pyte/screens.py
+
+
+class AnsiGraphics(enum.Enum):
+    # https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_(Select_Graphic_Rendition)_parameters
+    RESET = 0
+    REVERSE = 7
+
+
+def ansi_sequence(n):
+    # https://en.wikipedia.org/wiki/ANSI_escape_code#CSI_(Control_Sequence_Introducer)_sequences
+    return f'\033[{n.value}m'
+
+
+def pyte_char_to_ansi(ch: Char) -> str:
+    """
+    Render a single isolated ``pyte.screens.Char`` using ANSI escape sequences
+
+    https://pyte.readthedocs.io/en/latest/api.html#pyte.screens.Char
+    """
+    chunks = []
+
+    if ch.reverse:
+        chunks.append(ansi_sequence(AnsiGraphics.RESET))
+        chunks.append(ansi_sequence(AnsiGraphics.REVERSE))
+
+    chunks.append(ch.data)
+
+    if ch.reverse:
+        chunks.append(ansi_sequence(AnsiGraphics.RESET))
+
+    return ''.join(chunks)
+
+
+def ansi_display(screen):
+    """
+    A (mostly identical) fork of ``pyte.screens.Screen.display``
+    that uses ``pyte_char_to_ansi`` rather than ``line[x].data``.
+    """
+    def render(line: StaticDefaultDict[int, Char]) -> Generator[str, None, None]:
+        is_wide_char = False
+        for x in range(screen.columns):
+            if is_wide_char:  # Skip stub
+                is_wide_char = False
+                continue
+            char = line[x].data
+            assert sum(map(wcwidth, char[1:])) == 0
+            is_wide_char = wcwidth(char[0]) == 2
+            yield pyte_char_to_ansi(line[x])
+
+    return ["".join(render(screen.buffer[y])) for y in range(screen.lines)]
+
+
+def dump(screen, frame_number):
+    print(f'[{screen.columns}x{screen.lines}] Frame {frame_number}:')
+    print(f'+{"-" * screen.columns}+')
+    for line in ansi_display(screen):
+        print(f'|{line}|')
+    print(f'+{"-" * screen.columns}+')
+    print(flush=True)
+
+
+def get_runtime_nanos():
+    return time.clock_gettime_ns(time.CLOCK_MONOTONIC)
+
+
+def create_rhythmic_dumper(screen):
+    nano_before = get_runtime_nanos()
+    frame_number = 1
+
+    screen.dirty.clear()
+
+    def dump_or_not(last=False):
+        nonlocal nano_before
+        nonlocal frame_number
+
+        nano_now = get_runtime_nanos()
+
+        # For CI robustness, we want to:
+        # 1. Display 1 frame per second at most
+        # 2. Never display two identical consecutive frames
+        # 3. Only show an all-empty frame if it is not the first frame
+        #    or the last frame
+        # 4. Always show at least one frame (overruling 3.)
+        if last or nano_now - nano_before >= 1_000_000_000:
+            if (frame_number == 1 and last) or (screen.dirty and not ((frame_number == 1 or last) and ''.join(ansi_display(screen)).isspace())):
+                dump(screen, frame_number=frame_number)
+                screen.dirty.clear()
+                frame_number += 1
+            nano_before = nano_now
+
+    return dump_or_not
+
+
+if __name__ == "__main__":
+    if len(sys.argv) <= 1:
+        progname_py = os.path.basename(sys.argv[0])
+        sys.exit(f'usage: python3 {progname_py} COMMAND [ARG ..]')
+
+    COLUMNS = 90
+    LINES = 20
+
+    screen = pyte.Screen(COLUMNS, LINES)
+    stream = pyte.ByteStream(screen)
+
+    dump_or_not = create_rhythmic_dumper(screen)
+
+    p_pid, master_fd = pty.fork()
+    if p_pid == 0:  # Child.
+        env = os.environ.copy()
+        env.update(dict(TERM="linux", COLUMNS=str(COLUMNS), LINES=str(LINES)))
+        os.execvpe(sys.argv[1], sys.argv[1:], env=env)
+
+    while True:
+        try:
+            readables, _w, _x = select.select(
+                [master_fd], [], [], 0.1)
+        except (KeyboardInterrupt,  # Stop right now!
+                ValueError):        # Nothing to read.
+            break
+
+        dump_or_not()
+
+        if not readables:
+            continue
+
+        try:
+            data = os.read(master_fd, 1024)
+        except OSError:
+            break
+
+        if not data:
+            break
+
+        stream.feed(data)
+
+        dump_or_not()
+
+    os.kill(p_pid, signal.SIGTERM)
+
+    dump_or_not(last=True)

recordings/get_back_in_sync.sh

@@ -22,17 +22,12 @@ fi
 
 ./record.sh
 
-for i in actual-*.png ; do
-    cp -v "${i}" "${i/actual/expected}"
-done
-
-if type -P zopflipng &>/dev/null; then
-    for i in expected-*.png ; do
-        # https://github.com/google/zopfli
-        zopflipng -y "${i}" "${i}"
-    done
-fi
+cp actual.txt expected.txt
 
-git add expected-*.png
+git add expected.txt
 
-EDITOR=true git commit -m 'recordings: Sync expected-*.png images'
+if git diff --cached --exit-code >/dev/null ; then
+    echo 'Already in sync, good.'
+else
+    EDITOR=true git commit -m 'recordings: Sync expected.txt'
+fi