Work on getting token references for composite tokens

.changeset/violet-lemons-greet.md

@@ -0,0 +1,5 @@
+---
+'@primer/primitives': minor
+---
+
+Enable token references in composite tokens

scripts/buildTokens.ts

@@ -1,4 +1,4 @@
-import type {Config} from 'style-dictionary/types'
+import type {Config, LogConfig} from 'style-dictionary/types'
 import {PrimerStyleDictionary} from '../src/primerStyleDictionary.js'
 import {copyFromDir} from '../src/utilities/index.js'
 import {deprecatedJson, css, docJson, fallbacks, styleLint} from '../src/platforms/index.js'
@@ -12,6 +12,14 @@ import {themes} from './themes.config.js'
 import fs from 'fs'
 import {getFallbackTheme} from './utilities/getFallbackTheme.js'
 
+const log: LogConfig = {
+  warnings: 'disabled', // 'warn' | 'error' | 'disabled'
+  verbosity: 'silent', // 'default' | 'silent' | 'verbose'
+  errors: {
+    brokenReferences: 'throw', // 'throw' | 'console'
+  },
+}
+
 /**
  * getStyleDictionaryConfig
  * @param filename output file name without extension
@@ -29,13 +37,7 @@ const getStyleDictionaryConfig: StyleDictionaryConfigGenerator = (
 ): Config => ({
   source, // build the special formats
   include,
-  log: {
-    warnings: 'disabled', // 'warn' | 'error' | 'disabled'
-    verbosity: 'silent', // 'default' | 'silent' | 'verbose'
-    errors: {
-      brokenReferences: 'throw', // 'throw' | 'console'
-    },
-  },
+  log,
   platforms: Object.fromEntries(
     Object.entries({
       css: css(`css/${filename}.css`, options.prefix, options.buildPath, {
@@ -64,6 +66,7 @@ export const buildDesignTokens = async (buildOptions: ConfigGeneratorOptions): P
       const extendedSD = await PrimerStyleDictionary.extend({
         source: [...source, ...include], // build the special formats
         include,
+        log,
         platforms: {
           css: css(`internalCss/${filename}.css`, buildOptions.prefix, buildOptions.buildPath, {
             themed: true,

src/formats/cssAdvanced.ts

@@ -1,6 +1,7 @@
 import type {TransformedToken, FormatFn, FormatFnArguments, FormattingOptions} from 'style-dictionary/types'
 import {format} from 'prettier'
-import {fileHeader, formattedVariables, sortByName} from 'style-dictionary/utils'
+import {fileHeader, sortByName} from 'style-dictionary/utils'
+import getFormattedVariables from './utilities/getFormattedVariables.js'
 
 const wrapWithSelector = (css: string, selector: string | false): string => {
   // return without selector
@@ -74,7 +75,7 @@ export const cssAdvanced: FormatFn = async ({
     // early abort if no matches
     if (!filteredDictionary.allTokens.length) continue
     // add tokens into root
-    const css = formattedVariables({
+    const css = getFormattedVariables({
       format: 'css',
       dictionary: filteredDictionary,
       outputReferences,

src/formats/utilities/createPropertyFormatterWithRef.ts

@@ -0,0 +1,248 @@
+/*
+ * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with
+ * the License. A copy of the License is located at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+import type {Dictionary, FormattingOptions, OutputReferences, TransformedToken} from 'style-dictionary/types'
+import {getReferences, usesReferences} from 'style-dictionary/utils'
+
+/**
+ * @typedef {import('../../../types/DesignToken.d.ts').TransformedToken} TransformedToken
+ * @typedef {import('../../../types/DesignToken.d.ts').Dictionary} Dictionary
+ * @typedef {import('../../../types/File.d.ts').FormattingOptions} Formatting
+ * @typedef {import('../../../types/Format.d.ts').OutputReferences} OutputReferences
+ */
+
+/**
+ * @type {Formatting}
+ */
+const defaultFormatting = {
+  prefix: '',
+  commentStyle: 'long',
+  commentPosition: 'inline',
+  indentation: '',
+  separator: ' =',
+  suffix: ';',
+}
+
+/**
+ * Split a string comment by newlines and
+ * convert to multi-line comment if necessary
+ * @param {string} toRetToken
+ * @param {string} comment
+ * @param {Formatting} options
+ * @returns {string}
+ */
+export function addComment(toRetToken: string, comment: string, options: FormattingOptions) {
+  const {commentStyle, indentation} = options
+  let {commentPosition} = options
+
+  const commentsByNewLine = comment.split('\n')
+  if (commentsByNewLine.length > 1) {
+    commentPosition = 'above'
+  }
+
+  let processedComment
+  switch (commentStyle) {
+    case 'short':
+      if (commentPosition === 'inline') {
+        processedComment = `// ${comment}`
+      } else {
+        processedComment = commentsByNewLine.reduce((acc, curr) => `${acc}${indentation}// ${curr}\n`, '')
+        // remove trailing newline
+        processedComment = processedComment.replace(/\n$/g, '')
+      }
+      break
+    case 'long':
+      if (commentsByNewLine.length > 1) {
+        processedComment = commentsByNewLine.reduce(
+          (acc, curr) => `${acc}${indentation} * ${curr}\n`,
+          `${indentation}/**\n`,
+        )
+        processedComment += `${indentation} */`
+      } else {
+        processedComment = `${commentPosition === 'above' ? indentation : ''}/* ${comment} */`
+      }
+      break
+  }
+
+  if (commentPosition === 'above') {
+    // put the comment above the token if it's multi-line or if commentStyle ended with -above
+    toRetToken = `${processedComment}\n${toRetToken}`
+  } else {
+    toRetToken = `${toRetToken} ${processedComment}`
+  }
+
+  return toRetToken
+}
+
+/**
+ * Creates a function that can be used to format a token. This can be useful
+ * to use as the function on `dictionary.allTokens.map`. The formatting
+ * is configurable either by supplying a `format` option or a `formatting` object
+ * which uses: prefix, indentation, separator, suffix, and commentStyle.
+ * @memberof module:formatHelpers
+ * @name createPropertyFormatter
+ * @example
+ * ```javascript
+ * import { propertyFormatNames } from 'style-dictionary/enums';
+ *
+ * StyleDictionary.registerFormat({
+ *   name: 'myCustomFormat',
+ *   format: function({ dictionary, options }) {
+ *     const { outputReferences } = options;
+ *     const formatProperty = createPropertyFormatter({
+ *       outputReferences,
+ *       dictionary,
+ *       format: propertyFormatNames.css
+ *     });
+ *     return dictionary.allTokens.map(formatProperty).join('\n');
+ *   }
+ * });
+ * ```
+ * @param {Object} options
+ * @param {OutputReferences} [options.outputReferences] - Whether or not to output references. You will want to pass this from the `options` object sent to the format function.
+ * @param {boolean} [options.outputReferenceFallbacks] - Whether or not to output css variable fallback values when using output references. You will want to pass this from the `options` object sent to the format function.
+ * @param {Dictionary} options.dictionary - The dictionary object sent to the format function
+ * @param {string} [options.format] - Available formats are: 'css', 'sass', 'less', and 'stylus'. If you want to customize the format and can't use one of those predefined formats, use the `formatting` option
+ * @param {Formatting} [options.formatting] - Custom formatting properties that define parts of a declaration line in code. The configurable strings are: `prefix`, `indentation`, `separator`, `suffix`, `lineSeparator`, `fileHeaderTimestamp`, `header`, `footer`, `commentStyle` and `commentPosition`. Those are used to generate a line like this: `${indentation}${prefix}${token.name}${separator} ${prop.value}${suffix}`. The remaining formatting options are used for the fileHeader helper.
+ * @param {boolean} [options.themeable] [false] - Whether tokens should default to being themeable.
+ * @param {boolean} [options.usesDtcg] [false] - Whether DTCG token syntax should be uses.
+ * @returns {(token: import('../../../types/DesignToken.d.ts').TransformedToken) => string}
+ */
+export default function createPropertyFormatterWithRef({
+  outputReferences = false,
+  outputReferenceFallbacks = false,
+  dictionary,
+  format,
+  formatting = {},
+  themeable = false,
+  usesDtcg = false,
+}: {
+  outputReferences?: OutputReferences
+  outputReferenceFallbacks?: boolean
+  dictionary: Dictionary
+  format?: string
+  formatting?: FormattingOptions
+  themeable?: boolean
+  usesDtcg?: boolean
+}) {
+  /** @type {Formatting} */
+  const formatDefaults: FormattingOptions = {}
+  switch (format) {
+    case 'css':
+      formatDefaults.prefix = '--'
+      formatDefaults.indentation = '  '
+      formatDefaults.separator = ':'
+      break
+  }
+  const mergedOptions = {
+    ...defaultFormatting,
+    ...formatDefaults,
+    ...formatting,
+  }
+  const {prefix, commentStyle, indentation, separator, suffix} = mergedOptions
+  const {tokens, unfilteredTokens} = dictionary
+  return function (token: TransformedToken) {
+    let toRetToken = `${indentation}${prefix}${token.name}${separator} `
+    let value = usesDtcg ? token.$value : token.value
+    const originalValue = usesDtcg ? token.original.$value : token.original.value
+    const shouldOutputRef =
+      usesReferences(originalValue) &&
+      (typeof outputReferences === 'function' ? outputReferences(token, {dictionary, usesDtcg}) : outputReferences)
+    /**
+     * A single value can have multiple references either by interpolation:
+     * "value": "{size.border.width.value} solid {color.border.primary.value}"
+     * or if the value is an object:
+     * "value": {
+     *    "size": "{size.border.width.value}",
+     *    "style": "solid",
+     *    "color": "{color.border.primary.value"}
+     * }
+     * This will see if there are references and if there are, replace
+     * the resolved value with the reference's name.
+     */
+    if (shouldOutputRef) {
+      // Formats that use this function expect `value` to be a string
+      // or else you will get '[object Object]' in the output
+      const refs = getReferences(originalValue, tokens, {unfilteredTokens, warnImmediately: false}, [])
+      // original can either be an object value, which requires transitive value transformation in web CSS formats
+      // or a different (primitive) type, meaning it can be stringified.
+      const originalIsObject = typeof originalValue === 'object' && originalValue !== null
+
+      if (!originalIsObject) {
+        // TODO: find a better way to deal with object-value tokens and outputting refs
+        // e.g. perhaps it is safer not to output refs when the value is transformed to a non-object
+        // for example for CSS-like formats we always flatten to e.g. strings
+
+        // when original is object value, we replace value by matching ref.value and putting a var instead.
+        // Due to the original.value being an object, it requires transformation, so undoing the transformation
+        // by replacing value with original.value is not possible. (this is the early v3 approach to outputting refs)
+
+        // when original is string value, we replace value by matching original.value and putting a var instead
+        // this is more friendly to transitive transforms that transform the string values (v4 way of outputting refs)
+        value = originalValue
+      } else {
+        if (token.$type === 'border') {
+          const transformedValues = value.split(' ')
+          value = ['width', 'style', 'color']
+            .map((prop, index) => {
+              if (
+                originalValue[prop].startsWith('{') &&
+                refs.find(ref => ref.path.join('.') === originalValue[prop].replace(/[{}]/g, ''))?.isSource === true
+              ) {
+                return originalValue[prop]
+              }
+              return transformedValues[index]
+            })
+            .join(' ')
+        }
+      }
+      /* eslint-disable-next-line github/array-foreach */
+      refs.forEach(ref => {
+        // value should be a string that contains the resolved reference
+        // because Style Dictionary resolved this in the resolution step.
+        // Here we are undoing that by replacing the value with
+        // the reference's name
+        if (Object.hasOwn(ref, `${usesDtcg ? '$' : ''}value`) && Object.hasOwn(ref, 'name')) {
+          const refVal = usesDtcg ? ref.$value : ref.value
+          const replaceFunc = () => {
+            if (format === 'css') {
+              if (outputReferenceFallbacks) {
+                return `var(${prefix}${ref.name}, ${refVal})`
+              } else {
+                return `var(${prefix}${ref.name})`
+              }
+            } else {
+              return `${prefix}${ref.name}`
+            }
+          }
+          // TODO: add test
+          // technically speaking a reference can be made to a number or boolean token, in this case we stringify it first
+          const regex = new RegExp(`{${ref.path.join('\\.')}(\\.\\$?value)?}`, 'g')
+          value = `${value}`.replace(regex, replaceFunc)
+        }
+      })
+    }
+    toRetToken += value
+
+    const themeableToken = typeof token.themeable === 'boolean' ? token.themeable : themeable
+    if (format === 'sass' && themeableToken) {
+      toRetToken += ' !default'
+    }
+
+    toRetToken += suffix
+    const comment = token.$description ?? token.comment
+    if (comment && commentStyle !== 'none') {
+      toRetToken = addComment(toRetToken, comment, mergedOptions as FormattingOptions)
+    }
+    return toRetToken
+  }
+}

src/formats/utilities/getFormattedVariables.ts

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with
+ * the License. A copy of the License is located at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+import type {Dictionary, OutputReferences} from 'style-dictionary/types'
+import {sortByReference} from 'style-dictionary/utils'
+import createPropertyFormatterWithRef from './createPropertyFormatterWithRef.js'
+
+const defaultFormatting = {
+  lineSeparator: '\n',
+}
+
+export default function getFormattedVariables({
+  format,
+  dictionary,
+  outputReferences = false,
+  outputReferenceFallbacks,
+  formatting = {},
+  themeable = false,
+  usesDtcg = false,
+}: {
+  format: string
+  dictionary: Dictionary
+  outputReferences?: OutputReferences
+  outputReferenceFallbacks?: boolean
+  formatting?: {
+    lineSeparator?: string
+  }
+  themeable?: boolean
+  usesDtcg?: boolean
+}) {
+  // typecast, we know that by know the tokens have been transformed
+  let allTokens = dictionary.allTokens
+  const tokens = dictionary.tokens
+
+  const {lineSeparator} = Object.assign({}, defaultFormatting, formatting)
+
+  // Some languages are imperative, meaning a variable has to be defined
+  // before it is used. If `outputReferences` is true, check if the token
+  // has a reference, and if it does send it to the end of the array.
+  // We also need to account for nested references, a -> b -> c. They
+  // need to be defined in reverse order: c, b, a so that the reference always
+  // comes after the definition
+  if (outputReferences) {
+    // note: using the spread operator here so we get a new array rather than
+    // mutating the original
+    allTokens = [...allTokens].sort(sortByReference(tokens, {unfilteredTokens: dictionary.unfilteredTokens, usesDtcg}))
+  }
+
+  return allTokens
+    .map(
+      createPropertyFormatterWithRef({
+        outputReferences,
+        outputReferenceFallbacks,
+        dictionary,
+        format,
+        formatting,
+        themeable,
+        usesDtcg,
+      }),
+    )
+    .filter(function (strVal) {
+      return !!strVal
+    })
+    .join(lineSeparator)
+}