From 1fb6d38853b4254b96e0f31396e9e7f728e3934b Mon Sep 17 00:00:00 2001 From: Alec Delaney Date: Wed, 13 Mar 2024 16:06:01 -0400 Subject: [PATCH 1/6] Add quotes to make command in Makefile for Windows setups --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index f332aa7..84876f9 100644 --- a/Makefile +++ b/Makefile @@ -45,9 +45,9 @@ endif .PHONY: test test: - -@${MAKE} test-prep --no-print-directory - -@${MAKE} test-run --no-print-directory - -@${MAKE} test-clean --no-print-directory + -@"${MAKE}" test-prep --no-print-directory + -@"${MAKE}" test-run --no-print-directory + -@"${MAKE}" test-clean --no-print-directory .PHONY: test-run: From 9d67112b17815faf5328a2a278bacf388794a037 Mon Sep 17 00:00:00 2001 From: Alec Delaney Date: Thu, 14 Mar 2024 00:56:18 -0400 Subject: [PATCH 2/6] Change subcommand to board-id for current command --- circfirm/cli/current.py | 4 ++-- tests/cli/test_cli_current.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/circfirm/cli/current.py b/circfirm/cli/current.py index c37462b..8aa6e3a 100644 --- a/circfirm/cli/current.py +++ b/circfirm/cli/current.py @@ -30,8 +30,8 @@ def cli() -> None: """Check the information about the currently connected board.""" -@cli.command(name="id") -def current_id() -> None: +@cli.command(name="board-id") +def current_board_id() -> None: """Get the board ID of the currently connected board.""" click.echo(get_board_info()[0]) diff --git a/tests/cli/test_cli_current.py b/tests/cli/test_cli_current.py index e3c8b25..26d7542 100644 --- a/tests/cli/test_cli_current.py +++ b/tests/cli/test_cli_current.py @@ -19,7 +19,7 @@ def test_current() -> None: """Tests the current name and version commands.""" # Test when connected in CIRCUITPY mode - result = RUNNER.invoke(cli, ["current", "id"]) + result = RUNNER.invoke(cli, ["current", "board-id"]) assert result.exit_code == 0 assert result.output == "feather_m4_express\n" @@ -31,5 +31,5 @@ def test_current() -> None: @tests.helpers.as_bootloader def test_current_in_bootloader() -> None: """Tests the current command whenn connected in bootloader mode.""" - result = RUNNER.invoke(cli, ["current", "id"]) + result = RUNNER.invoke(cli, ["current", "board-id"]) assert result.exit_code != 0 From 173529d1f58d145776fad3dc5f9a61d11c4a616b Mon Sep 17 00:00:00 2001 From: Alec Delaney Date: Thu, 14 Mar 2024 00:57:18 -0400 Subject: [PATCH 3/6] Change boards regex from match to search --- circfirm/cli/query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circfirm/cli/query.py b/circfirm/cli/query.py index cc8a70e..b33e2ab 100644 --- a/circfirm/cli/query.py +++ b/circfirm/cli/query.py @@ -54,7 +54,7 @@ def query_boards(regex: str) -> None: ) for board in boards: board_id = board.strip() - result = re.match(regex, board_id) + result = re.search(regex, board_id) if result: click.echo(board_id) From 3e3b3f59f6b7097fe4c8c590855b5ff28e9dfa72 Mon Sep 17 00:00:00 2001 From: Alec Delaney Date: Thu, 14 Mar 2024 01:38:37 -0400 Subject: [PATCH 4/6] Add documentation for commands --- docs/commands/cache.rst | 42 ++++++++++++++++++++++ docs/commands/config.rst | 41 +++++++++++++++++++++ docs/commands/current.rst | 24 +++++++++++++ docs/commands/install.rst | 25 +++++++++++++ docs/commands/query.rst | 76 +++++++++++++++++++++++++++++++++++++++ docs/commands/update.rst | 36 +++++++++++++++++++ 6 files changed, 244 insertions(+) diff --git a/docs/commands/cache.rst b/docs/commands/cache.rst index 1f89410..680006e 100644 --- a/docs/commands/cache.rst +++ b/docs/commands/cache.rst @@ -4,3 +4,45 @@ Caching Versions ================ + +You can interact with the local cache of CircuitPython firmware versions using ``circfirm cache``. + +See ``circfirm cache --help`` and ``circfirm cache [command] --help`` for more information on commands. + +Saving Versions +--------------- + +You can save versions of the CircuitPython firmware using ``circfirm cache save``. + +.. code-block:: shell + + # Save the CircuitPython 8.0.0 firmware for the feather_m4_express board + circfirm cache save feather_m4_express 8.0.0 + +Listing Versions +---------------- + +You can list cached versions of the CircuitPython firmware using ``circfirm cache list``. + +.. code-block:: shell + + # List all the firmware versions + circfirm cache list + + # List all the firmware versions for the feather_m4_express board + circfirm cache list --board-id feather_m4_express + +Clearing the Cache +------------------ + +You can clearr cached firmware versions using ``circfirm cache clear``. + +You can also specify what should be cleared in terms of board IDs, versions, and languages. + +.. code-block:: shell + + # Clear the cache + circfirm cache clear + + # Clear the cache of French versions of the feather_m4_express + circfirm cache clear --board-id feather_m4_express --language fr diff --git a/docs/commands/config.rst b/docs/commands/config.rst index a0a8b92..0e1685f 100644 --- a/docs/commands/config.rst +++ b/docs/commands/config.rst @@ -4,3 +4,44 @@ Configuring the CLI =================== + +You can modify the CLI configurations settings using ``circfirm config``. + +See ``circfirm config --help`` and ``circfirm config [command] --help`` for more information on commands. + +View Settings +------------- + +You can view configuration settings using ``circfirm config view``. + +You can view all the settings, a subset, or just a specific one. Subsettings are accessed using period separators. + +.. code-block:: shell + + # View all the configuration settings + circfirm config view + + # View a specific setting + circfirm config view output.supporting.silence + +Edit Settings +------------- + +You can edit configuration settings using ``circfirm config view``. + +You must edit a specific setting at a time. Subsettings are accessed using period separators. + +.. code-block:: shell + + # Edit a configuration settings with a value + circfirm config edit output.supporting.silence true + +Reset Settings +-------------- + +You can reset the configuration settings to the default using ``circfirm config reset``. + +.. code-block:: shell + + # Reset the configuration settings to the default + circfirm config reset diff --git a/docs/commands/current.rst b/docs/commands/current.rst index c0cd26a..2f70dda 100644 --- a/docs/commands/current.rst +++ b/docs/commands/current.rst @@ -4,3 +4,27 @@ Checking the Current Version ============================ + +You can get information about a currently connected board using ``circfirm current``. + +See ``circfirm current --help`` and ``circfirm current [command] --help`` for more information on commands. + +Getting the Board ID +-------------------- + +You can get the board ID of the currently connected board using ``circfirm current board-id``. + +.. code-block:: shell + + # Get the board ID of the connected board + circfirm current board-id + +Getting the Firmware Version +---------------------------- + +You can get the CircuitPython version of the currently connected board using ``circfirm current version``. + +.. code-block:: shell + + # Get the firmware version of the connected board + circfirm current verson diff --git a/docs/commands/install.rst b/docs/commands/install.rst index 8d05d38..ba4d89a 100644 --- a/docs/commands/install.rst +++ b/docs/commands/install.rst @@ -4,3 +4,28 @@ Installing Specific Versions ============================ + +You can install different CircuitPython versions to a connected board using ``circfirm install``. + +See ``circfirm install --help`` for more information. + +The board should be connected so it shows up as a CIRCUITPY USB drive (or equivalently named), +where the board ID will be read from the ``boot_out.txt`` file. The CLI will then prompt you to set the +board into bootloader mode, after which the selected CircuitPython version will be installed on +the board. + +If you wish to skip the step where the board ID is collected and simply connected the board in +bootloader mode, you can do so and simply use the ``--board-id`` option to provide the board ID. + +You can specify a language using the ``--language`` option - the default is US English. + +.. code-block:: shell + + # Install CircuitPython 8.0.0 on the connected board + circfirm install 8.0.0 + + # Install the French translation of CircuitPython on the connected board + circfirm install 8.0.0 --language fr + + # Install CircuitPython 8.0.0 on the connected Adafruit QT Py ESP32 Pico (in bootloader mode) + circfirm install 8.0.0 --board-id adafruit_qtpy_esp32_pico diff --git a/docs/commands/query.rst b/docs/commands/query.rst index cb5ac1f..d81131c 100644 --- a/docs/commands/query.rst +++ b/docs/commands/query.rst @@ -4,3 +4,79 @@ Querying Boards and Versions ============================ + +You can query for valid board IDs and CircuitPython versions from online resources using ``circfirm query``. + +See ``circfirm query --help`` and ``circfirm query [command] --help`` for more information on commands. + +Querying Board IDs +------------------ + +You can query a list of valid board IDs from the CircuitPython GitHub repository using ``circfirm query boards``. + +You can use the ``--regex`` option to further select boards from the list matching a provided regex pattern. +The pattern will be searched for **ANYWHERE** in the board ID (e.g., "hello" **would** match "123hello123") unless +the pattern specifies otherwise. + +.. note:: + + Querying board IDs communicates with GitHub, which can only be done 60 times per hour unauthenticated. + If you plan to make frequent use of this command consider adding a GitHub token to the configuration + settings (``circfirm config edit token.github ``). + +.. code-block:: shell + + # List all board IDs + circfirm query boards + + # List all board IDs containing the phrase "pico" + circfirm query boards --regex pico + +Querying Board Versions +----------------------- + +You can query a list of CircuitPython versions for a board ID in the official AWS S3 bucket of firmware +using ``circfirm query version``. + +You can use the ``--regex`` option to further select versions from the list matching a provided regex pattern. +The pattern will be searched for **FROM THE BEGINNING** in the board ID (e.g., "hello" **would not** match "123hello123"). +This is done to make matching entire version sets more convenient ("8\.2\..+" matches the entirety of 8.2 and all associated +minor, alpha, beta and release candidate versions). + +You can also set the language using the ``--language`` option, which can affect the list of available versions. + +.. code-block:: shell + + # List all available versions for the Feather M4 Express + circfirm query versions feather_m4_express + + # List all available French versions for the Feather M4 Express + circfirm query versions feather_m4_express --language fr + + # List all versions in the 8.2.X set for the Feather M4 Express + circfirm query versions feather_m4_express --regex 8\.2\..+ + +Query the Latest Version +------------------------ + +You can query the latest version of CircuitPython use ``circfirm query latest`` + +This command take a board ID as an argument, as this may have an effect on available CircuitPython versions. +For simplicity, if information is not needed about a specific board but rather the CircuitPython project as +a whole, the default board ID ``raspberry_pi_pico`` is used, as it is an actively supported board. + +You can also set the language using the ``--language`` option, which can affect the list of available versions. + +If you would like to include pre-release versions as potential latest versions, you can use the +``--pre-release`` flag. + +.. code-block:: shell + + # Get the latest version of CircuitPython + circfirm query latest + + # Get the latest version of CircuitPython for the Feather M4 Express + circfirm query latest feather_m4_express + + # Get the latest version of CircuitPython for the Feather M4 Express, including pre-releases + circfirm query latest feather_m4_express --pre-release diff --git a/docs/commands/update.rst b/docs/commands/update.rst index 7ce6134..5f99386 100644 --- a/docs/commands/update.rst +++ b/docs/commands/update.rst @@ -4,3 +4,39 @@ Updating to the Latest Version ============================== + +You can update the CircuitPython version of a connected board using ``circfirm update``. + +See ``circfirm update --help`` for more information. + +The board should be connected so it shows up as a CIRCUITPY USB drive (or equivalently named), +where the board ID will be read from the ``boot_out.txt`` file. The CLI will then prompt you to set the +board into bootloader mode, after which the selected CircuitPython version will be installed on +the board. + +If you wish to skip the step where the board ID is collected and simply connected the board in +bootloader mode, you can do so and simply use the ``--board-id`` option to provide the board ID. + +You can specify a language using the ``--language`` option - the default is US English. + +If you would like to include pre-releases as potential update versions, you can use the +``--pre-release`` flag. + +.. note:: + + This command will not update the board if the detected version of CircuitPython on the connected + board is greater than or equal to updated version. + +.. code-block:: shell + + # Update CircuitPython on the connected board + circfirm update + + # Update the French translation of CircuitPython on the connected board + circfirm --language fr + + # Update CircuitPython 8.0.0 on the connected Adafruit QT Py ESP32 Pico (in bootloader mode) + circfirm update --board-id adafruit_qtpy_esp32_pico + + # Update CircuitPython on the connected board, considering pre-release versions + circfirm update --pre-release From 6977634d99cf6e88b179d9efc566accc1d14cdd0 Mon Sep 17 00:00:00 2001 From: Alec Delaney Date: Thu, 14 Mar 2024 01:47:35 -0400 Subject: [PATCH 5/6] Update boards subcommand name for query command --- circfirm/backend/github.py | 2 +- circfirm/cli/query.py | 8 ++++---- docs/commands/query.rst | 6 +++--- tests/backend/test_backend_github.py | 6 +++--- tests/cli/test_cli_query.py | 16 ++++++++-------- tests/helpers.py | 2 +- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/circfirm/backend/github.py b/circfirm/backend/github.py index 8b76f51..7b1180c 100644 --- a/circfirm/backend/github.py +++ b/circfirm/backend/github.py @@ -55,7 +55,7 @@ def get_rate_limit() -> Tuple[int, int, datetime.datetime]: return available, total, reset_time -def get_board_list(token: str) -> List[str]: +def get_board_id_list(token: str) -> List[str]: """Get a list of CircuitPython boards.""" boards = set() headers = BASE_REQUESTS_HEADERS.copy() diff --git a/circfirm/cli/query.py b/circfirm/cli/query.py index b33e2ab..1ffe022 100644 --- a/circfirm/cli/query.py +++ b/circfirm/cli/query.py @@ -23,9 +23,9 @@ def cli(): """Query things like latest versions and board lists.""" -@cli.command(name="boards") +@cli.command(name="board-ids") @click.option("-r", "--regex", default=".*", help="Regex pattern to use for board IDs") -def query_boards(regex: str) -> None: +def query_board_ids(regex: str) -> None: """Query the local CircuitPython board list.""" settings = circfirm.cli.get_settings() gh_token = settings["token"]["github"] @@ -41,11 +41,11 @@ def query_boards(regex: str) -> None: if do_output: boards = circfirm.cli.announce_and_await( "Fetching boards list", - circfirm.backend.github.get_board_list, + circfirm.backend.github.get_board_id_list, args=(gh_token,), ) else: - boards = circfirm.backend.github.get_board_list(gh_token) + boards = circfirm.backend.github.get_board_id_list(gh_token) except ValueError as err: raise click.ClickException(err.args[0]) except requests.ConnectionError as err: diff --git a/docs/commands/query.rst b/docs/commands/query.rst index d81131c..17f7eb1 100644 --- a/docs/commands/query.rst +++ b/docs/commands/query.rst @@ -12,7 +12,7 @@ See ``circfirm query --help`` and ``circfirm query [command] --help`` for more i Querying Board IDs ------------------ -You can query a list of valid board IDs from the CircuitPython GitHub repository using ``circfirm query boards``. +You can query a list of valid board IDs from the CircuitPython GitHub repository using ``circfirm query board-ids``. You can use the ``--regex`` option to further select boards from the list matching a provided regex pattern. The pattern will be searched for **ANYWHERE** in the board ID (e.g., "hello" **would** match "123hello123") unless @@ -27,10 +27,10 @@ the pattern specifies otherwise. .. code-block:: shell # List all board IDs - circfirm query boards + circfirm query board-ids # List all board IDs containing the phrase "pico" - circfirm query boards --regex pico + circfirm query board-ids --regex pico Querying Board Versions ----------------------- diff --git a/tests/backend/test_backend_github.py b/tests/backend/test_backend_github.py index 1950696..ed05355 100644 --- a/tests/backend/test_backend_github.py +++ b/tests/backend/test_backend_github.py @@ -21,13 +21,13 @@ def test_get_board_list() -> None: """Tests the ability of the backend to get the board list.""" # Test successful retrieval token = os.environ["GH_TOKEN"] - board_list = circfirm.backend.github.get_board_list(token) - expected_board_list = tests.helpers.get_boards_from_git() + board_list = circfirm.backend.github.get_board_id_list(token) + expected_board_list = tests.helpers.get_board_ids_from_git() assert board_list == expected_board_list # Test unsuccessful retrieval with pytest.raises(ValueError): - circfirm.backend.github.get_board_list("badtoken") + circfirm.backend.github.get_board_id_list("badtoken") def test_get_rate_limit() -> None: diff --git a/tests/cli/test_cli_query.py b/tests/cli/test_cli_query.py index f82244b..fddc525 100644 --- a/tests/cli/test_cli_query.py +++ b/tests/cli/test_cli_query.py @@ -26,11 +26,11 @@ def simulate_no_connection(arg: str) -> NoReturn: raise requests.ConnectionError -def test_query_boards(monkeypatch: pytest.MonkeyPatch) -> None: +def test_query_board_ids(monkeypatch: pytest.MonkeyPatch) -> None: """Tests the ability to query the boards using the CLI.""" # Test an unauthenticated request with supporting text - boards = tests.helpers.get_boards_from_git() - pre_expected_output = "".join([f"{board}\n" for board in boards]) + board_ids = tests.helpers.get_board_ids_from_git() + pre_expected_output = "".join([f"{board}\n" for board in board_ids]) expected_output = "\n".join( [ "Boards list will now be synchronized with the git repository.", @@ -40,7 +40,7 @@ def test_query_boards(monkeypatch: pytest.MonkeyPatch) -> None: ] ) - result = RUNNER.invoke(cli, ["query", "boards"]) + result = RUNNER.invoke(cli, ["query", "board-ids"]) assert result.exit_code == 0 assert result.output == expected_output @@ -51,14 +51,14 @@ def test_query_boards(monkeypatch: pytest.MonkeyPatch) -> None: assert result.exit_code == 0 result = RUNNER.invoke(cli, ["config", "edit", "output.supporting.silence", "true"]) assert result.exit_code == 0 - result = RUNNER.invoke(cli, ["query", "boards"]) + result = RUNNER.invoke(cli, ["query", "board-ids"]) assert result.exit_code == 0 assert result.output == pre_expected_output # Test a request with a faulty token result = RUNNER.invoke(cli, ["config", "edit", "token.github", "badtoken"]) assert result.exit_code == 0 - result = RUNNER.invoke(cli, ["query", "boards"]) + result = RUNNER.invoke(cli, ["query", "board-ids"]) assert result.exit_code != 0 result = RUNNER.invoke(cli, ["config", "reset"]) @@ -66,9 +66,9 @@ def test_query_boards(monkeypatch: pytest.MonkeyPatch) -> None: # Tests failure when cannot fetch results due to no network connection monkeypatch.setattr( - circfirm.backend.github, "get_board_list", simulate_no_connection + circfirm.backend.github, "get_board_id_list", simulate_no_connection ) - result = RUNNER.invoke(cli, ["query", "boards"]) + result = RUNNER.invoke(cli, ["query", "board-ids"]) assert result.exit_code != 0 assert ( result.output.split("\n")[-2] diff --git a/tests/helpers.py b/tests/helpers.py index 805f3fd..952453e 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -134,7 +134,7 @@ def copy_firmwares() -> None: ) -def get_boards_from_git() -> List[str]: +def get_board_ids_from_git() -> List[str]: """Get a list of board IDs from the sandbox git repository.""" ports_path = pathlib.Path("tests/sandbox/circuitpython") board_paths = ports_path.glob("ports/*/boards/*") From 5a64c424c01b21cc7114eca3c1b9d7cc7b7c9633 Mon Sep 17 00:00:00 2001 From: Alec Delaney Date: Thu, 14 Mar 2024 13:00:59 -0400 Subject: [PATCH 6/6] Add documentation for infrastructure pages --- docs/infrastructure/ci.rst | 36 +++++++++++++++++++++++++++++++ docs/infrastructure/codecheck.rst | 30 ++++++++++++++++++++++++++ docs/infrastructure/docs.rst | 8 +++++++ docs/infrastructure/tests.rst | 11 ++++++++++ 4 files changed, 85 insertions(+) diff --git a/docs/infrastructure/ci.rst b/docs/infrastructure/ci.rst index 495658c..d88477e 100644 --- a/docs/infrastructure/ci.rst +++ b/docs/infrastructure/ci.rst @@ -4,3 +4,39 @@ CI/CD Pipeline ============== + +.. _build-ci: + +Build CI +-------- + +When code is pushed to the repository or a pull request is submitted, it will +trigger a GitHub Actions run of the build CI, which can be found in +``.github/workflows/push.yml``. This build CI performs the following checks: + +- Code quality +- Packaging build +- Code testing +- Code coverage + +You can read more about the code checks performed :doc:`here ` + +CodeQL CI +--------- + +Similar to the build CI, a GitHub Actions performs a CodeQL check on the code +whenever code is pushed to the repository or a pull request is submitted. +Additionally, it runs this action daily to ensure best practices and look for +issues as CodeQL is improved. + +.. _publish-ci: + +Publish CI +---------- + +When a release is made on GitHub for the project, it builds the project using +the Python ``build`` package. Before building, the publish CI pushes the +release tag to the ``VERSION`` file. The CI then uses ``build`` to generate +both source and binary distributions of the project. The CU then uploads +these distributions to PyPI for download through tools such as ``pipx`` and +``pip``. diff --git a/docs/infrastructure/codecheck.rst b/docs/infrastructure/codecheck.rst index bb7d384..232db8e 100644 --- a/docs/infrastructure/codecheck.rst +++ b/docs/infrastructure/codecheck.rst @@ -4,3 +4,33 @@ Code Checking ============= + +Code checks are performed using ``pre-commit``, which runs git pre-commit hooks on all files. + +If you are developing the ``circfirm`` codebase, you can install these hooks into git by +runing ``pre-commit install``. + +You can manually perform a check of the code, which includes all of the items below, by +running ``make check``. + +Linting & Formatting +-------------------- + +Linting and formatting are done using ``ruff`` pre-commit hooks, as well as some of +the basic ``pre-commit`` ones provided by the project. + +You can manually run linting and formatting checks using ``make lint`` and ``make format`` +respectively. + +Secret Detection +---------------- + +Yelp's ``detect-secrets`` tool runs to try and prevent secrets from being accidentally +committed and shared. + +Open Source Compliance +---------------------- + +The FSF's REUSE tool runs on the entire repository to make sure all files are properly +labelled with an open source license and conform to REUSE standards, which makes it +easier for other people to reuse any part of it. diff --git a/docs/infrastructure/docs.rst b/docs/infrastructure/docs.rst index 7c51249..104a8be 100644 --- a/docs/infrastructure/docs.rst +++ b/docs/infrastructure/docs.rst @@ -4,3 +4,11 @@ Documentation ============= + +Documentation is generated by Sphinx and a number of Sphinx extensions. + +This documentation is pushed with every ``git push`` using a repository +webhook to ReadTheDocs so others can view it online. + +You can manually test building the documentation during development by +running ``make docs``. diff --git a/docs/infrastructure/tests.rst b/docs/infrastructure/tests.rst index c9fc834..482582c 100644 --- a/docs/infrastructure/tests.rst +++ b/docs/infrastructure/tests.rst @@ -4,3 +4,14 @@ Testing & Code Coverage ======================= + +The project contains test code for testing compliance of the tool with +expected behavior. Testing is done using coverage.py, which uses pytest +to actually run the tests. Coverage.py then analyzes the code that was +run in both the source code and tests themselves to calculate the code +coverage metric. + +In the :ref:`build CI ` for pull requests, this code coverage +metric is uploaded to Codecov, which then comments on the pull request +to help identify whether additional tests are needed based on the +changes.