From fa25845127464bfa50a1dc0dbe37a55a8aba36f0 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Wed, 21 Feb 2024 19:28:17 -0800 Subject: [PATCH 01/13] Frontmatter validation via docusaurus plugin See: - https://docusaurus.io/docs/markdown-features#front-matter - https://ajv.js.org/guide/getting-started.html#basic-data-validation --- .github/resource/frontmatter.schema.json | 23 +++++++++++++++++++++++ docusaurus.config.js | 24 ++++++++++++++++++++++++ package.json | 3 ++- pnpm-lock.yaml | 6 +++--- 4 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 .github/resource/frontmatter.schema.json diff --git a/.github/resource/frontmatter.schema.json b/.github/resource/frontmatter.schema.json new file mode 100644 index 000000000..b6955c08c --- /dev/null +++ b/.github/resource/frontmatter.schema.json @@ -0,0 +1,23 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://ozone.apache.org/frontmatter.schema.json", + "title": "Frontmatter", + "description": "Validates an allowed subset of Docusaurus docs frontmatter.", + "type": [ "object", "null" ], + "properties": { + "sidebar_label": { + "type": "string" + }, + "slug": { + "description": "Only use slug to specify the base URL for documentation.", + "type": "string", + "enum": [ "/" ] + }, + "draft": { + "description": "Makes pages for incomplete features only visible in development mode.", + "type": "boolean" + } + }, + "additionalProperties": false +} + diff --git a/docusaurus.config.js b/docusaurus.config.js index c7b2f5e85..163b90dd9 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -1,6 +1,8 @@ // @ts-check // Note: type annotations allow type checking and IDEs autocompletion +import Ajv from 'ajv'; + const {themes} = require('prism-react-renderer'); const lightCodeTheme = themes.github; const darkCodeTheme = themes.dracula; @@ -34,6 +36,28 @@ const config = { locales: ['en'], }, + markdown: { + // Validate markdown frontmatter against a more restrictive schema than what Docusaurus allows. + // This ensures all pages are using a minimal set of consistent keys. + parseFrontMatter: async (params) => { + // Reuse the default parser. + const result = await params.defaultParseFrontMatter(params); + + // Validate frontmatter against the schema. + const frontmatterSchema = require('./.github/resource/frontmatter.schema.json'); + const ajv = new Ajv(); + const validate = ajv.compile(frontmatterSchema); + const isValid = validate(result.frontMatter); + + if (!isValid) { + console.error('Frontmatter validation error in ' + params.filePath + ':', validate.errors); + process.exit(1); + } + + return result; + }, + }, + presets: [ [ 'classic', diff --git a/package.json b/package.json index 981ceed20..5f6ce356f 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,8 @@ "clsx": "^2.1.0", "prism-react-renderer": "^2.3.1", "react": "^18.2.0", - "react-dom": "^18.2.0" + "react-dom": "^18.2.0", + "ajv": "^8.12.0" }, "devDependencies": { "@docusaurus/module-type-aliases": "3.1.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7da582edc..9ed6b6d86 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -43,6 +43,9 @@ devDependencies: '@docusaurus/module-type-aliases': specifier: 3.1.1 version: 3.1.1(react-dom@18.2.0)(react@18.2.0) + ajv: + specifier: ^8.12.0 + version: 8.12.0 cspell: specifier: ^8.2.1 version: 8.4.1 @@ -3443,7 +3446,6 @@ packages: json-schema-traverse: 1.0.0 require-from-string: 2.0.2 uri-js: 4.4.1 - dev: false /algoliasearch-helper@3.16.2(algoliasearch@4.22.1): resolution: {integrity: sha512-Yl/Gu5Cq4Z5s/AJ0jR37OPI1H3+z7PHz657ibyaXgMOaWvPlZ3OACN13N+7HCLPUlB0BN+8BtmrG/CqTilowBA==} @@ -6146,7 +6148,6 @@ packages: /json-schema-traverse@1.0.0: resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} - dev: false /json5@2.2.3: resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==} @@ -8297,7 +8298,6 @@ packages: /require-from-string@2.0.2: resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} engines: {node: '>=0.10.0'} - dev: false /require-like@0.1.2: resolution: {integrity: sha512-oyrU88skkMtDdauHDuKVrgR+zuItqr6/c//FXzvmxRGMexSDc6hNvJInGW3LL46n+8b50RykrvwSUIIQH2LQ5A==} From addb6837c25a253bb5b9d5e2700f51ca9757bdf4 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Wed, 21 Feb 2024 19:42:29 -0800 Subject: [PATCH 02/13] Add sidebar check --- .github/resource/category.schema.json | 16 ++++++++ .github/scripts/sidebar.sh | 47 +++++++++++++++++++++ .github/workflows/static.yml | 19 +++++++++ package.json | 3 +- pnpm-lock.yaml | 59 +++++++++++++++++++-------- 5 files changed, 125 insertions(+), 19 deletions(-) create mode 100644 .github/resource/category.schema.json create mode 100755 .github/scripts/sidebar.sh diff --git a/.github/resource/category.schema.json b/.github/resource/category.schema.json new file mode 100644 index 000000000..f235ddcd5 --- /dev/null +++ b/.github/resource/category.schema.json @@ -0,0 +1,16 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://ozone.apache.org/category.schema.json", + "title": "Category", + "description": "Validates an allowed subset of keys in Docusaurus _category_.yml files.", + "type": "object", + "properties": { + "label": { + "type": "string" + } + }, + "required": [ + "label" + ], + "additionalProperties": false +} diff --git a/.github/scripts/sidebar.sh b/.github/scripts/sidebar.sh new file mode 100755 index 000000000..34425214c --- /dev/null +++ b/.github/scripts/sidebar.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +# Validates docusaurus _category_.yml files used to configure the docs sidebar. +# Each docs subdirectory should have a _category_.yml file, and it must follow the defined schema. + +rc=0 + +root="$(git rev-parse --show-toplevel)" +schema="$root"/.github/resource/category.schema.json + +check_dir() { + dir="$1" + category_file="$dir/_category_.yml" + if [ ! -f "$category_file" ]; then + echo "_category_.yml is required for docs subdirectory $dir" 1>&2 + rc=1 + elif ! pnpm ajv validate --errors=text -s "$schema" -d "$category_file" 1>/dev/null; then + # Ensure the category file contains only allowed keys. + rc=1 + fi +} + +for child in $(find "$root"/docs/*); do + if [ -d "$child" ]; then + check_dir "$child" + fi +done + +exit "$rc" + diff --git a/.github/workflows/static.yml b/.github/workflows/static.yml index 7c5b4cc18..017ed55df 100644 --- a/.github/workflows/static.yml +++ b/.github/workflows/static.yml @@ -55,3 +55,22 @@ jobs: if: ${{ failure() }} run: | cat ${{ env.resource_dir }}/spelling_tips.md >> $GITHUB_STEP_SUMMARY + sidebar: + runs-on: ubuntu-latest + steps: + - name: Checkout project + uses: actions/checkout@v4 + - name: Enable corepack + run: | + corepack enable + - name: Use Node.js ${{ env.node_version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ env.node_version }} + cache: pnpm + - name: Install pnpm dependencies + run: pnpm install --dev + - name: Run docs sidebar check + working-directory: ${{ env.script_dir }} + run: | + ./sidebar.sh diff --git a/package.json b/package.json index 5f6ce356f..d3b3b7d07 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,8 @@ "@cspell/dict-docker": "^1.1.7", "@cspell/dict-java": "^5.0.6", "@cspell/dict-markdown": "^2.0.1", - "@cspell/dict-shell": "^1.0.6" + "@cspell/dict-shell": "^1.0.6", + "ajv-cli": "^5.0.0" }, "browserslist": { "production": [ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9ed6b6d86..24fbf346f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,6 +14,9 @@ dependencies: '@mdx-js/react': specifier: ^3.0.0 version: 3.0.0(@types/react@18.2.48)(react@18.2.0) + ajv: + specifier: ^8.12.0 + version: 8.12.0 clsx: specifier: ^2.1.0 version: 2.1.0 @@ -43,9 +46,9 @@ devDependencies: '@docusaurus/module-type-aliases': specifier: 3.1.1 version: 3.1.1(react-dom@18.2.0)(react@18.2.0) - ajv: - specifier: ^8.12.0 - version: 8.12.0 + ajv-cli: + specifier: ^5.0.0 + version: 5.0.0 cspell: specifier: ^8.2.1 version: 8.4.1 @@ -3404,6 +3407,24 @@ packages: indent-string: 4.0.0 dev: false + /ajv-cli@5.0.0: + resolution: {integrity: sha512-LY4m6dUv44HTyhV+u2z5uX4EhPYTM38Iv1jdgDJJJCyOOuqB8KtZEGjPZ2T+sh5ZIJrXUfgErYx/j3gLd3+PlQ==} + hasBin: true + peerDependencies: + ts-node: '>=9.0.0' + peerDependenciesMeta: + ts-node: + optional: true + dependencies: + ajv: 8.12.0 + fast-json-patch: 2.2.1 + glob: 7.2.3 + js-yaml: 3.14.1 + json-schema-migrate: 2.0.0 + json5: 2.2.3 + minimist: 1.2.8 + dev: true + /ajv-formats@2.1.1(ajv@8.12.0): resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==} peerDependencies: @@ -3528,7 +3549,6 @@ packages: resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==} dependencies: sprintf-js: 1.0.3 - dev: false /argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} @@ -3710,7 +3730,6 @@ packages: dependencies: balanced-match: 1.0.2 concat-map: 0.0.1 - dev: false /brace-expansion@2.0.1: resolution: {integrity: sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==} @@ -4053,7 +4072,6 @@ packages: /concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} - dev: false /config-chain@1.1.13: resolution: {integrity: sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==} @@ -4982,6 +5000,10 @@ packages: /extend@3.0.2: resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} + /fast-deep-equal@2.0.1: + resolution: {integrity: sha512-bCK/2Z4zLidyB4ReuIsvALH6w31YfAQDmXMqMx6FyfHqvBxtjC0eRumeSu4Bs3XtXwpyIywtSTrVT99BxY1f9w==} + dev: true + /fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -5000,6 +5022,13 @@ packages: merge2: 1.4.1 micromatch: 4.0.5 + /fast-json-patch@2.2.1: + resolution: {integrity: sha512-4j5uBaTnsYAV5ebkidvxiLUYOwjQ+JSFljeqfTxCrH9bDmlCQaOJFS84oDJ2rAXZq2yskmk3ORfoP9DCwqFNig==} + engines: {node: '>= 0.4.0'} + dependencies: + fast-deep-equal: 2.0.1 + dev: true + /fast-json-stable-stringify@2.1.0: resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} @@ -5229,7 +5258,6 @@ packages: /fs.realpath@1.0.0: resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} - dev: false /fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} @@ -5317,7 +5345,6 @@ packages: minimatch: 3.1.2 once: 1.4.0 path-is-absolute: 1.0.1 - dev: false /global-directory@4.0.1: resolution: {integrity: sha512-wHTUcDUoZ1H5/0iVqEudYW4/kAlN5cZ3j/bXn0Dpbizl9iaUVeWSHqiOjsgk6OW2bkLclbBjzewBz6weQ1zA2Q==} @@ -5835,7 +5862,6 @@ packages: dependencies: once: 1.4.0 wrappy: 1.0.2 - dev: false /inherits@2.0.3: resolution: {integrity: sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==} @@ -5843,7 +5869,6 @@ packages: /inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} - dev: false /ini@1.3.8: resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} @@ -6117,7 +6142,6 @@ packages: dependencies: argparse: 1.0.10 esprima: 4.0.1 - dev: false /js-yaml@4.1.0: resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==} @@ -6143,6 +6167,12 @@ packages: /json-parse-even-better-errors@2.3.1: resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} + /json-schema-migrate@2.0.0: + resolution: {integrity: sha512-r38SVTtojDRp4eD6WsCqiE0eNDt4v1WalBXb9cyZYw9ai5cGtBwzRNWjHzJl38w6TxFkXAIA7h+fyX3tnrAFhQ==} + dependencies: + ajv: 8.12.0 + dev: true + /json-schema-traverse@0.4.1: resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} @@ -6153,7 +6183,6 @@ packages: resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==} engines: {node: '>=6'} hasBin: true - dev: false /jsonfile@6.1.0: resolution: {integrity: sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==} @@ -6969,7 +6998,6 @@ packages: resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} dependencies: brace-expansion: 1.1.11 - dev: false /minimatch@9.0.3: resolution: {integrity: sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==} @@ -6980,7 +7008,6 @@ packages: /minimist@1.2.8: resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} - dev: false /minipass@7.0.4: resolution: {integrity: sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==} @@ -7130,7 +7157,6 @@ packages: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} dependencies: wrappy: 1.0.2 - dev: false /onetime@5.1.2: resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==} @@ -7319,7 +7345,6 @@ packages: /path-is-absolute@1.0.1: resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} engines: {node: '>=0.10.0'} - dev: false /path-is-inside@1.0.2: resolution: {integrity: sha512-DUWJr3+ULp4zXmol/SZkFf3JGsS9/SIv+Y3Rt93/UjPpDpklB5f1er4O3POIbUuUJ3FXgqte2Q7SrU6zAqwk8w==} @@ -8721,7 +8746,6 @@ packages: /sprintf-js@1.0.3: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} - dev: false /srcset@4.0.0: resolution: {integrity: sha512-wvLeHgcVHKO8Sc/H/5lkGreJQVeYMm9rlmt8PuR1xE31rIuXhuzznUUqAt8MqLhB3MqJdFzlNAfpcWnxiFUcPw==} @@ -9437,7 +9461,6 @@ packages: /wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} - dev: false /write-file-atomic@3.0.3: resolution: {integrity: sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q==} From 779bbbf2b8151e4be54e70abfabb12edfe9ce7c6 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Wed, 21 Feb 2024 19:44:14 -0800 Subject: [PATCH 03/13] Remove extra newlines --- .github/scripts/sidebar.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/scripts/sidebar.sh b/.github/scripts/sidebar.sh index 34425214c..2a9b7b8ec 100755 --- a/.github/scripts/sidebar.sh +++ b/.github/scripts/sidebar.sh @@ -16,7 +16,6 @@ # specific language governing permissions and limitations # under the License. - # Validates docusaurus _category_.yml files used to configure the docs sidebar. # Each docs subdirectory should have a _category_.yml file, and it must follow the defined schema. @@ -32,7 +31,6 @@ check_dir() { echo "_category_.yml is required for docs subdirectory $dir" 1>&2 rc=1 elif ! pnpm ajv validate --errors=text -s "$schema" -d "$category_file" 1>/dev/null; then - # Ensure the category file contains only allowed keys. rc=1 fi } @@ -44,4 +42,3 @@ for child in $(find "$root"/docs/*); do done exit "$rc" - From be055f52c32d2e2c8aacc284ea0afeb1c3e13ce3 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Wed, 7 Feb 2024 13:43:57 -0800 Subject: [PATCH 04/13] Fix frontamtter keys in documents --- docs/07-system-internals/02-data-operations/04-hsync.md | 3 +-- docs/07-system-internals/07-features/05-ratis-streaming.md | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/07-system-internals/02-data-operations/04-hsync.md b/docs/07-system-internals/02-data-operations/04-hsync.md index 4c1b5f40f..866e7b49b 100644 --- a/docs/07-system-internals/02-data-operations/04-hsync.md +++ b/docs/07-system-internals/02-data-operations/04-hsync.md @@ -1,6 +1,5 @@ --- -# Page is accessible by direct link only in production site builds. It can be included in the sidebar when the feature is complete. -unlisted: true +draft: true sidebar_label: HFlush and HSync --- diff --git a/docs/07-system-internals/07-features/05-ratis-streaming.md b/docs/07-system-internals/07-features/05-ratis-streaming.md index 6276509e7..8f270dc71 100644 --- a/docs/07-system-internals/07-features/05-ratis-streaming.md +++ b/docs/07-system-internals/07-features/05-ratis-streaming.md @@ -1,6 +1,5 @@ --- -# Page is accessible by direct link only in production site builds. It can be included in the sidebar when the feature is complete. -unlisted: true +draft: true sidebar_label: Ratis Streaming --- From b3cecd69978fde585ab931fa8c27edc14dd37b58 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Wed, 21 Feb 2024 20:05:50 -0800 Subject: [PATCH 05/13] Include mdx in spelling check --- .github/scripts/spelling.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/spelling.sh b/.github/scripts/spelling.sh index c97131a63..12bf1c8e1 100755 --- a/.github/scripts/spelling.sh +++ b/.github/scripts/spelling.sh @@ -21,7 +21,7 @@ root="$(git rev-parse --show-toplevel)" rc=0 echo 'Checking document content...' -pnpm cspell lint --root="$root" --no-progress --show-context '**/*.md' '**/_category_.yml' || rc="$?" +pnpm cspell lint --root="$root" --no-progress --show-context '**/*.md' '**/*.mdx' '**/_category_.yml' || rc="$?" echo 'Checking file names...' find "$root"/docs "$root"/src/pages | pnpm cspell --no-progress --show-context stdin://'File Name' || rc="$?" From bf3ef4ddcd676d227a297255699512ab317cdcd6 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Wed, 21 Feb 2024 20:06:05 -0800 Subject: [PATCH 06/13] Massive speedup in sidebar check by batching to ajv --- .github/scripts/sidebar.sh | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/scripts/sidebar.sh b/.github/scripts/sidebar.sh index 2a9b7b8ec..198e5506e 100755 --- a/.github/scripts/sidebar.sh +++ b/.github/scripts/sidebar.sh @@ -24,21 +24,22 @@ rc=0 root="$(git rev-parse --show-toplevel)" schema="$root"/.github/resource/category.schema.json -check_dir() { - dir="$1" - category_file="$dir/_category_.yml" - if [ ! -f "$category_file" ]; then - echo "_category_.yml is required for docs subdirectory $dir" 1>&2 - rc=1 - elif ! pnpm ajv validate --errors=text -s "$schema" -d "$category_file" 1>/dev/null; then - rc=1 - fi -} - +# Make sure all docs directories have a category sidebar file. for child in $(find "$root"/docs/*); do if [ -d "$child" ]; then - check_dir "$child" + category_file="$child/_category_.yml" + if [ ! -f "$category_file" ]; then + echo "_category_.yml is required for docs subdirectory $child" 1>&2 + rc=1 + fi fi done +[ "$rc" = 0 ] || exit $rc + +# If all category files are present, make sure they follow the schema. +if ! pnpm ajv validate --errors=text -s "$schema" -d "$root/docs/**/_category_.yml" 1>/dev/null; then + rc=1 +fi + exit "$rc" From 235ea4a6c9b85150f6af1bcf44bd0f7a5c8c2b74 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Wed, 21 Feb 2024 20:08:43 -0800 Subject: [PATCH 07/13] Fix ratis streaming link --- docs/07-system-internals/02-data-operations/01-write.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/07-system-internals/02-data-operations/01-write.md b/docs/07-system-internals/02-data-operations/01-write.md index 2190f766c..be2eba498 100644 --- a/docs/07-system-internals/02-data-operations/01-write.md +++ b/docs/07-system-internals/02-data-operations/01-write.md @@ -26,7 +26,7 @@ Trace every part of a write request from beginning to end. This includes: - For Ratis: - Include topology choices of which datanodes to use - Include failover handling - - For [EC](../features/erasure-coding) and [Ratis Streaming](../features/ratis-streaming), link to their feature pages. + - For [EC](../features/erasure-coding) and Ratis streaming, link to their feature pages. - Client allocating more blocks if needed - Client committing to OM - OM checking the current namespace From b3a6d5e4afcf2fe0b1bec8311b26e3f1b2dd7adb Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Mon, 4 Mar 2024 16:59:03 -0800 Subject: [PATCH 08/13] Add more detail in schema check comments --- .github/scripts/sidebar.sh | 2 ++ .github/workflows/static.yml | 2 +- docusaurus.config.js | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/scripts/sidebar.sh b/.github/scripts/sidebar.sh index 198e5506e..56af06bbb 100755 --- a/.github/scripts/sidebar.sh +++ b/.github/scripts/sidebar.sh @@ -18,6 +18,8 @@ # Validates docusaurus _category_.yml files used to configure the docs sidebar. # Each docs subdirectory should have a _category_.yml file, and it must follow the defined schema. +# The schema is more restrictive than what Docusaurus allows, and can be used to disallow keys or require all category +# files to define the same keys. rc=0 diff --git a/.github/workflows/static.yml b/.github/workflows/static.yml index ce15c36ef..3d39912ac 100644 --- a/.github/workflows/static.yml +++ b/.github/workflows/static.yml @@ -83,7 +83,7 @@ jobs: working-directory: ${{ env.script_dir }} run: | ./markdownlint.sh - sidebar: + check-sidebar: runs-on: ubuntu-latest steps: - name: Checkout project diff --git a/docusaurus.config.js b/docusaurus.config.js index 163b90dd9..ee14ebf5d 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -39,6 +39,7 @@ const config = { markdown: { // Validate markdown frontmatter against a more restrictive schema than what Docusaurus allows. // This ensures all pages are using a minimal set of consistent keys. + // It can also be used to require all pages to define certain markdown frontmatter keys. parseFrontMatter: async (params) => { // Reuse the default parser. const result = await params.defaultParseFrontMatter(params); From 1ea372bd8108cd2f90b4150e626b1dbccf4c0700 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Mon, 4 Mar 2024 19:08:22 -0800 Subject: [PATCH 09/13] Clearer error message from sidebar script --- .github/scripts/sidebar.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/sidebar.sh b/.github/scripts/sidebar.sh index 56af06bbb..4d8e29f1b 100755 --- a/.github/scripts/sidebar.sh +++ b/.github/scripts/sidebar.sh @@ -40,7 +40,7 @@ done [ "$rc" = 0 ] || exit $rc # If all category files are present, make sure they follow the schema. -if ! pnpm ajv validate --errors=text -s "$schema" -d "$root/docs/**/_category_.yml" 1>/dev/null; then +if ! pnpm ajv validate -s "$schema" -d "$root/docs/**/_category_.yml" 1>/dev/null; then rc=1 fi From f3ee87133bebd3cc2b3a035bbb2819a3a09dad51 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Mon, 4 Mar 2024 19:14:56 -0800 Subject: [PATCH 10/13] Undo change to spelling script --- .github/scripts/spelling.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/spelling.sh b/.github/scripts/spelling.sh index 12bf1c8e1..c97131a63 100755 --- a/.github/scripts/spelling.sh +++ b/.github/scripts/spelling.sh @@ -21,7 +21,7 @@ root="$(git rev-parse --show-toplevel)" rc=0 echo 'Checking document content...' -pnpm cspell lint --root="$root" --no-progress --show-context '**/*.md' '**/*.mdx' '**/_category_.yml' || rc="$?" +pnpm cspell lint --root="$root" --no-progress --show-context '**/*.md' '**/_category_.yml' || rc="$?" echo 'Checking file names...' find "$root"/docs "$root"/src/pages | pnpm cspell --no-progress --show-context stdin://'File Name' || rc="$?" From 503da60849629a47176c739e2d116b3faf8a1926 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Mon, 4 Mar 2024 19:39:27 -0800 Subject: [PATCH 11/13] Updates after reviewing diff --- .github/resource/category.schema.json | 1 + .github/workflows/static.yml | 2 +- docs/07-system-internals/02-data-operations/04-hsync.md | 1 + docs/07-system-internals/07-features/05-ratis-streaming.md | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/resource/category.schema.json b/.github/resource/category.schema.json index f235ddcd5..afa67b906 100644 --- a/.github/resource/category.schema.json +++ b/.github/resource/category.schema.json @@ -6,6 +6,7 @@ "type": "object", "properties": { "label": { + "description": "The name of this section shown in the docs sidebar.", "type": "string" } }, diff --git a/.github/workflows/static.yml b/.github/workflows/static.yml index 3d39912ac..5f972c71c 100644 --- a/.github/workflows/static.yml +++ b/.github/workflows/static.yml @@ -101,4 +101,4 @@ jobs: - name: Run docs sidebar check working-directory: ${{ env.script_dir }} run: | - ./sidebar.sh \ No newline at end of file + ./sidebar.sh diff --git a/docs/07-system-internals/02-data-operations/04-hsync.md b/docs/07-system-internals/02-data-operations/04-hsync.md index 71d97ed91..f561324ef 100644 --- a/docs/07-system-internals/02-data-operations/04-hsync.md +++ b/docs/07-system-internals/02-data-operations/04-hsync.md @@ -1,4 +1,5 @@ --- +# Page is available only in the development server. It can be included in production builds when the feature is complete. draft: true sidebar_label: HFlush and HSync --- diff --git a/docs/07-system-internals/07-features/05-ratis-streaming.md b/docs/07-system-internals/07-features/05-ratis-streaming.md index 68dc325d4..1c898519d 100644 --- a/docs/07-system-internals/07-features/05-ratis-streaming.md +++ b/docs/07-system-internals/07-features/05-ratis-streaming.md @@ -1,4 +1,5 @@ --- +# Page is available only in the development server. It can be included in production builds when the feature is complete. draft: true sidebar_label: Ratis Streaming --- From 4f6f7edd384d1796af0d83b6f87195a98faeeede Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Mon, 4 Mar 2024 20:03:29 -0800 Subject: [PATCH 12/13] Print the json schema file path when errors occur --- .github/scripts/sidebar.sh | 1 + docusaurus.config.js | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/scripts/sidebar.sh b/.github/scripts/sidebar.sh index 4d8e29f1b..6b270946d 100755 --- a/.github/scripts/sidebar.sh +++ b/.github/scripts/sidebar.sh @@ -42,6 +42,7 @@ done # If all category files are present, make sure they follow the schema. if ! pnpm ajv validate -s "$schema" -d "$root/docs/**/_category_.yml" 1>/dev/null; then rc=1 + echo "Sidebar configuration validation failed against JSON schema $schema" 1>&2 fi exit "$rc" diff --git a/docusaurus.config.js b/docusaurus.config.js index ee14ebf5d..e51fcb2ad 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -45,13 +45,15 @@ const config = { const result = await params.defaultParseFrontMatter(params); // Validate frontmatter against the schema. - const frontmatterSchema = require('./.github/resource/frontmatter.schema.json'); + const schemaPath = './.github/resource/frontmatter.schema.json'; + const frontMatterSchema = require(schemaPath); const ajv = new Ajv(); - const validate = ajv.compile(frontmatterSchema); + const validate = ajv.compile(frontMatterSchema); const isValid = validate(result.frontMatter); if (!isValid) { - console.error('Frontmatter validation error in ' + params.filePath + ':', validate.errors); + console.error('Front matter validation error in', params.filePath + ':\n', validate.errors); + console.error('Front matter validation failed against JSON schema', schemaPath); process.exit(1); } From c785de5ee6566423297a1c033d0ba7ce6977bea5 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Mon, 4 Mar 2024 20:43:18 -0800 Subject: [PATCH 13/13] Fix spelling of "front matter" --- CONTRIBUTING.md | 6 +++--- cspell.yaml | 1 - docusaurus.config.js | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7211f18a5..9c2f1e2b7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -100,7 +100,7 @@ Docusaurus provides many options for laying out documentation pages and their me - File names and therefore generated URLs should all be `kebab-case`. -- Let Docusaurus automatically generate the page's title from its top level markdown heading instead of using `title` in the frontmatter. This is different from the sidebar label or URL slug. +- Let Docusaurus automatically generate the page's title from its top level markdown heading instead of using `title` in the front matter. This is different from the sidebar label or URL slug. - Use relative file paths for all links between pages. - This keeps page links pointing to the current version of the documentation. @@ -129,7 +129,7 @@ Docusaurus provides many options for laying out documentation pages and their me Object Store Bucket Layout ``` - - Sidebar labels are automatically generated from page titles. To use a different sidebar label, use the `sidebar_label` property in the page's frontmatter. + - Sidebar labels are automatically generated from page titles. To use a different sidebar label, use the `sidebar_label` property in the page's front matter. - Don't rely on Ozone specific acronyms in the sidebar. This makes docs navigation more beginner friendly. - For example, the pages on bucket layouts may be organized as: @@ -178,7 +178,7 @@ The file names and content of all markdown pages are checked for spelling mistak - Option 1: If the word is relevant for the whole Ozone project, add it to the `words` list in *cspell.yaml* so that it is considered valid. -- Option 2: If the word is only relevant for one specific page, add an [inline directive](https://cspell.org/configuration/document-settings/) as a comment in the markdown frontmatter of that page only. +- Option 2: If the word is only relevant for one specific page, add an [inline directive](https://cspell.org/configuration/document-settings/) as a comment in the markdown front matter of that page only. ### Updating Graphics diff --git a/cspell.yaml b/cspell.yaml index 98e176545..2d7b30c8e 100644 --- a/cspell.yaml +++ b/cspell.yaml @@ -78,7 +78,6 @@ words: - gpg - sed - mds -- frontmatter - javadoc # Misc words - acking diff --git a/docusaurus.config.js b/docusaurus.config.js index e51fcb2ad..dd1d60630 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -39,12 +39,12 @@ const config = { markdown: { // Validate markdown frontmatter against a more restrictive schema than what Docusaurus allows. // This ensures all pages are using a minimal set of consistent keys. - // It can also be used to require all pages to define certain markdown frontmatter keys. + // It can also be used to require all pages to define certain markdown front matter keys. parseFrontMatter: async (params) => { // Reuse the default parser. const result = await params.defaultParseFrontMatter(params); - // Validate frontmatter against the schema. + // Validate front matter against the schema. const schemaPath = './.github/resource/frontmatter.schema.json'; const frontMatterSchema = require(schemaPath); const ajv = new Ajv();