From ec9b4aca0ad78d769e7538d727c13dbe95ad2b6a Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Thu, 1 Aug 2024 23:48:52 +0200 Subject: [PATCH] macros: make `quote` a proper quasi-quoting operator (#1393) ## Summary `quote` now keeps the quoted block as is, which means that: * no symbols are bound or mixed in automatically from the enclosing scopes * identifiers in definition positions aren't turned into gensyms; they stay as is * `.gensym` and `.inject` pragmas within the quoted block don't affect the AST Symbols thus need to be bound or created explicitly, via `bindSym` and `genSym`, respectively. **This is a breaking change.** ## Details ### Motivation For The Change * symbols from the `quote`'s scope being bound is error-prone, leading to confusing compilation errors * the intention of documentation of `quote` already said it does quasi- quoting (even though it didn't) * the implementation relied on the intricacies of templates and template evaluation ### New Behaviour * quoted AST is not modified. No symbols are bound and no identifiers are turned into gensyms ### Implementation * `semQuoteAst` transforms the `quote` call into a call to the internal `quoteImpl` procedure * the pre-processed quoted AST is passed as the first arguments; the extracted unquoted expression are passed as the remaining arguments * the internal-only `evalToAst` magic procedure is used for evaluating the unquoted expressions. `newLit` cannot be used here, as the trees it produces for `object` values are only valid when all the type's fields are exported * placing the AST is of the evaluated unqouted expressions is handled in-VM, by `quoteImpl` ### Standard Library And Test Changes * multiple modules from the standard library relied on the previous symbol binding and gensym behaviour; they're changed to use `bindSym` or `genSym`. Outside-visible behaviour doesn't change * the `t7875.nim` test relied on the gensym behaviour. The definition outside the macro is not relevant to the issue the test guards against, so it can just be removed * the quoted ASTs in `tsizeof.nim` are wrapped in blocks in order to prevent the identifiers from colliding --- compiler/ast/ast_types.nim | 2 + compiler/sem/semexprs.nim | 99 ++++++++----------- compiler/sem/sempass2.nim | 2 + compiler/sem/varpartitions.nim | 2 +- compiler/vm/vmgen.nim | 9 +- lib/core/macros.nim | 27 ++++- lib/experimental/ast_pattern_matching.nim | 2 +- lib/js/asyncjs.nim | 3 +- lib/js/jsffi.nim | 49 ++++----- lib/std/jsonutils.nim | 4 +- lib/std/tasks.nim | 3 +- lib/std/wrapnils.nim | 6 +- tests/lang_callable/macros/t7875.nim | 5 +- tests/lang_callable/macros/tmacro6.nim | 2 +- .../defer/tdefer_malformed_many_children.nim | 2 +- tests/misc/tsizeof.nim | 21 ++-- 16 files changed, 135 insertions(+), 103 deletions(-) diff --git a/compiler/ast/ast_types.nim b/compiler/ast/ast_types.nim index 9269889125e..ff49afbb6a1 100644 --- a/compiler/ast/ast_types.nim +++ b/compiler/ast/ast_types.nim @@ -817,6 +817,8 @@ type mException, mBuiltinType, mSymOwner, mUncheckedArray, mGetImplTransf, mSymIsInstantiationOf, mNodeId, mPrivateAccess + mEvalToAst + # magics only used internally: mStrToCStr ## the backend-dependent string-to-cstring conversion diff --git a/compiler/sem/semexprs.nim b/compiler/sem/semexprs.nim index f476c717149..e98531714e9 100644 --- a/compiler/sem/semexprs.nim +++ b/compiler/sem/semexprs.nim @@ -2552,9 +2552,6 @@ proc expectString(c: PContext, n: PNode): string = else: localReport(c.config, n, reportSem rsemStringLiteralExpected) -proc newAnonSym(c: PContext; kind: TSymKind, info: TLineInfo): PSym = - result = newSym(kind, c.cache.idAnon, nextSymId c.idgen, getCurrOwner(c), info) - proc semExpandToAst(c: PContext, n: PNode): PNode = let macroCall = n[1] @@ -2606,14 +2603,11 @@ proc semExpandToAst(c: PContext, n: PNode, magicSym: PSym, else: result = semDirectOp(c, n, flags) -proc processQuotations(c: PContext; n: var PNode, op: string, - quotes: var seq[PNode], - ids: var seq[PNode]) = +proc processQuotations(c: PContext; n: PNode, op: string, call: PNode): PNode = template returnQuote(q) = - quotes.add q - n = newIdentNode(getIdent(c.cache, $quotes.len), n.info) - ids.add n - return + call.add q + # return a placeholder node. The integer represents the parameter index + return newTreeI(nkAccQuoted, n.info, newIntNode(nkIntLit, call.len - 3)) template handlePrefixOp(prefixed) = if prefixed[0].kind == nkIdent: @@ -2639,14 +2633,22 @@ proc processQuotations(c: PContext; n: var PNode, op: string, tempNode[0] = n[0] tempNode[1] = n[1] handlePrefixOp(tempNode) - of nkIdent: - if n.ident.s == "result": - n = ids[0] else: discard # xxx: raise an error + result = n for i in 0.. 0: - dummyTemplate[paramsPos] = newNodeI(nkFormalParams, n.info) - dummyTemplate[paramsPos].add: - getSysSym(c.graph, n.info, "untyped").newSymNode # return type - ids.add getSysSym(c.graph, n.info, "untyped").newSymNode # params type - ids.add c.graph.emptyNode # no default value - dummyTemplate[paramsPos].add newTreeI(nkIdentDefs, n.info, ids) - - var tmpl = semTemplateDef(c, dummyTemplate) - quotes[0] = tmpl[namePos] - # This adds a call to newIdentNode("result") as the first argument to the - # template call - let identNodeSym = getCompilerProc(c.graph, "newIdentNode") - # so that new Nim compilers can compile old macros.nim versions, we check for - # 'nil' here and provide the old fallback solution: - let identNode = if identNodeSym == nil: - newIdentNode(getIdent(c.cache, "newIdentNode"), n.info) - else: - identNodeSym.newSymNode - quotes[1] = newTreeI(nkCall, n.info, identNode, newStrNode(nkStrLit, "result")) - result = - c.semExpandToAst: - newTreeI(nkCall, n.info, - createMagic(c.graph, c.idgen, "getAst", mExpandToAst).newSymNode, - newTreeI(nkCall, n.info, quotes)) + # turn the quasi-quoted block into a call to the internal ``quoteImpl`` + # procedure + # future direction: implement this transformation in user code. The compiler + # only needs to provide an AST quoting facility (without quasi-quoting) + + let call = newNodeI(nkCall, n.info, 2) + call[0] = newSymNode(c.graph.getCompilerProc("quoteImpl")) + # extract the unquoted parts and append them to `call`: + let quoted = processQuotations(c, quotedBlock, op, call) + # the pre-processed AST of the quoted block is passed as the first argument: + call[1] = newTreeI(nkNimNodeLit, n.info, quoted) + call[1].typ = sysTypeFromName(c.graph, n.info, "NimNode") + + template ident(name: string): PNode = + newIdentNode(c.cache.getIdent(name), unknownLineInfo) + + # the unquoted expressions are wrapped in evalToAst calls. Use a qualified + # identifier in order to prevent user-defined evalToAst calls to be picked + let callee = newTree(nkDotExpr, ident("macros"), ident("evalToAst")) + for i in 2..`_ which avoids some issues with `quote`. + ## See also: + ## * `genasts `_ runnableExamples: macro check(ex: untyped) = # this is a simplified version of the check macro from the @@ -591,6 +592,30 @@ proc quote*(bl: typed, op = "``"): NimNode {.magic: "QuoteAst", noSideEffect.} = doAssert y == 3 bar2() +proc quoteImpl(n: NimNode, args: varargs[NimNode]): NimNode {.compilerproc, + compileTime.} = + ## Substitutes the placeholders in `n` with the corresponding AST from + ## `args`. Invoked by the compiler for implementating ``quote``. + proc aux(n: NimNode, args: openArray[NimNode]): NimNode = + case n.kind + of nnkAccQuoted: + if n[0].kind == nnkAccQuoted: + result = n[0] # an escaped accquoted tree + else: + result = args[n[0].intVal] # a placeholder + else: + result = n + for i in 0.. 0: # turn |NimSkull| outgoing exceptions into JavaScript errors let body = result.body + let reraise = bindSym("reraise") result.body = quote: try: `body` except CatchableError as e: # use .noreturn call to make sure `body` being an expression works - reraise(e) + `reraise`(e) let asyncPragma = quote: {.codegenDecl: "async function $2($3)".} diff --git a/lib/js/jsffi.nim b/lib/js/jsffi.nim index bd70c0277e9..d5360e6bff1 100644 --- a/lib/js/jsffi.nim +++ b/lib/js/jsffi.nim @@ -225,38 +225,36 @@ macro `.`*(obj: JsObject, field: untyped): JsObject = let obj = newJsObject() obj.a = 20 assert obj.a.to(int) == 20 + let helper = genSym(nskProc, "helper") if validJsName($field): let importString = "#." & $field result = quote do: - proc helper(o: JsObject): JsObject - {.importjs: `importString`, gensym.} - helper(`obj`) + proc `helper`(o: JsObject): JsObject {.importjs: `importString`.} + `helper`(`obj`) else: if not mangledNames.hasKey($field): mangledNames[$field] = $mangleJsName($field) let importString = "#." & mangledNames[$field] result = quote do: - proc helper(o: JsObject): JsObject - {.importjs: `importString`, gensym.} - helper(`obj`) + proc `helper`(o: JsObject): JsObject {.importjs: `importString`.} + `helper`(`obj`) macro `.=`*(obj: JsObject, field, value: untyped): untyped = ## Experimental dot accessor (set) for type JsObject. ## Sets the value of a property of name `field` in a JsObject `x` to `value`. + let helper = genSym(nskProc, "helper") if validJsName($field): let importString = "#." & $field & " = #" result = quote do: - proc helper(o: JsObject, v: auto) - {.importjs: `importString`, gensym.} - helper(`obj`, `value`) + proc `helper`(o: JsObject, v: auto) {.importjs: `importString`.} + `helper`(`obj`, `value`) else: if not mangledNames.hasKey($field): mangledNames[$field] = $mangleJsName($field) let importString = "#." & mangledNames[$field] & " = #" result = quote do: - proc helper(o: JsObject, v: auto) - {.importjs: `importString`, gensym.} - helper(`obj`, `value`) + proc `helper`(o: JsObject, v: auto) {.importjs: `importString`.} + `helper`(`obj`, `value`) macro `.()`*(obj: JsObject, field: untyped, @@ -284,10 +282,12 @@ macro `.()`*(obj: JsObject, if not mangledNames.hasKey($field): mangledNames[$field] = $mangleJsName($field) importString = "#." & mangledNames[$field] & "(@)" + + let helper = genSym(nskProc, "helper") result = quote: - proc helper(o: JsObject): JsObject - {.importjs: `importString`, gensym, discardable.} - helper(`obj`) + proc `helper`(o: JsObject): JsObject + {.importjs: `importString`, discardable.} + `helper`(`obj`) for idx in 0 ..< args.len: let paramName = newIdentNode("param" & $idx) result[0][3].add newIdentDefs(paramName, newIdentNode("JsObject")) @@ -304,10 +304,11 @@ macro `.`*[K: cstring, V](obj: JsAssoc[K, V], if not mangledNames.hasKey($field): mangledNames[$field] = $mangleJsName($field) importString = "#." & mangledNames[$field] + + let helper = genSym(nskProc, "helper") result = quote do: - proc helper(o: type(`obj`)): `obj`.V - {.importjs: `importString`, gensym.} - helper(`obj`) + proc `helper`(o: type(`obj`)): `obj`.V {.importjs: `importString`.} + `helper`(`obj`) macro `.=`*[K: cstring, V](obj: JsAssoc[K, V], field: untyped, @@ -321,10 +322,11 @@ macro `.=`*[K: cstring, V](obj: JsAssoc[K, V], if not mangledNames.hasKey($field): mangledNames[$field] = $mangleJsName($field) importString = "#." & mangledNames[$field] & " = #" + + let helper = genSym(nskProc, "helper") result = quote do: - proc helper(o: type(`obj`), v: `obj`.V) - {.importjs: `importString`, gensym.} - helper(`obj`, `value`) + proc `helper`(o: type(`obj`), v: `obj`.V) {.importjs: `importString`.} + `helper`(`obj`, `value`) macro `.()`*[K: cstring, V: proc](obj: JsAssoc[K, V], field: untyped, @@ -447,10 +449,11 @@ macro `{}`*(typ: typedesc, xs: varargs[untyped]): auto = body.add quote do: return `a` + let inner = genSym(nskProc, "inner") result = quote do: - proc inner(): `typ` {.gensym.} = + proc `inner`(): `typ` = `body` - inner() + `inner`() # Macro to build a lambda using JavaScript's `this` # from a proc, `this` being the first argument. diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim index 9fea21cc5c7..2aac6df8120 100644 --- a/lib/std/jsonutils.nim +++ b/lib/std/jsonutils.nim @@ -186,11 +186,11 @@ proc discKeyMatch[T](obj: T, json: JsonNode, key: static string): bool = macro discKeysMatchBodyGen(obj: typed, json: JsonNode, keys: static seq[string]): untyped = result = newStmtList() - let r = ident("result") + let match = bindSym("discKeyMatch") for key in keys: let keyLit = newLit key result.add quote do: - `r` = `r` and discKeyMatch(`obj`, `json`, `keyLit`) + result = result and `match`(`obj`, `json`, `keyLit`) proc discKeysMatch[T](obj: T, json: JsonNode, keys: static seq[string]): bool = result = true diff --git a/lib/std/tasks.nim b/lib/std/tasks.nim index ac18862228e..615a2cadcb3 100644 --- a/lib/std/tasks.nim +++ b/lib/std/tasks.nim @@ -183,8 +183,9 @@ macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkC ) + let cAlloc = bindSym("c_calloc") let scratchObjPtrType = quote do: - cast[ptr `scratchObjType`](c_calloc(csize_t 1, csize_t sizeof(`scratchObjType`))) + cast[ptr `scratchObjType`](`cAlloc`(csize_t 1, csize_t sizeof(`scratchObjType`))) let scratchLetSection = newLetStmt( scratchIdent, diff --git a/lib/std/wrapnils.nim b/lib/std/wrapnils.nim index 763af52fc0b..e7a1593d282 100644 --- a/lib/std/wrapnils.nim +++ b/lib/std/wrapnils.nim @@ -145,12 +145,14 @@ macro `??.`*(a: typed): Option = let lhs = genSym(nskVar, "lhs") let lhs2 = genSym(nskVar, "lhs") let body = process(a, lhs2, 0) + let optionSym = bindSym("Option") + let optionOpSym = bindSym("option") result = quote do: - var `lhs`: Option[type(`a`)] + var `lhs`: `optionSym`[type(`a`)] block: var `lhs2`: type(`a`) `body` - `lhs` = option(`lhs2`) + `lhs` = `optionOpSym`(`lhs2`) `lhs` template fakeDot*(a: Option, b): untyped = diff --git a/tests/lang_callable/macros/t7875.nim b/tests/lang_callable/macros/t7875.nim index 7b6e47b8657..648b8fe6d35 100644 --- a/tests/lang_callable/macros/t7875.nim +++ b/tests/lang_callable/macros/t7875.nim @@ -1,5 +1,5 @@ discard """ - nimout: "var mysym`gensym0: MyType[float32]" + nimout: "var mysym: MyType[float32]" joinable: false """ @@ -8,9 +8,6 @@ import macros type MyType[T] = object -# this is totally fine -var mysym: MyType[float32] - macro foobar(): untyped = let floatSym = bindSym"float32" diff --git a/tests/lang_callable/macros/tmacro6.nim b/tests/lang_callable/macros/tmacro6.nim index c65d34b6d7b..d9c7f9b4e81 100644 --- a/tests/lang_callable/macros/tmacro6.nim +++ b/tests/lang_callable/macros/tmacro6.nim @@ -1,6 +1,6 @@ discard """ errormsg: "expression '123' is of type 'int literal(123)' and has to be used (or discarded)" -line: 71 +line: 73 """ import macros diff --git a/tests/lang_stmts/defer/tdefer_malformed_many_children.nim b/tests/lang_stmts/defer/tdefer_malformed_many_children.nim index 324ba6b56d1..8d01ce15249 100644 --- a/tests/lang_stmts/defer/tdefer_malformed_many_children.nim +++ b/tests/lang_stmts/defer/tdefer_malformed_many_children.nim @@ -2,7 +2,7 @@ discard """ description: "`defer` must have exactly one child node (macro input)." errormsg: "illformed AST" file: "macros.nim" - line: 622 + line: 647 """ import std/macros diff --git a/tests/misc/tsizeof.nim b/tests/misc/tsizeof.nim index 909c35c17db..a71c42da491 100644 --- a/tests/misc/tsizeof.nim +++ b/tests/misc/tsizeof.nim @@ -29,10 +29,13 @@ doAssert mysize3 == 32 import macros, typetraits +proc wrapBlock(n: NimNode): NimNode = + result = newTree(nnkBlockStmt, newEmptyNode(), n) + macro testSizeAlignOf(args: varargs[untyped]): untyped = result = newStmtList() for arg in args: - result.add quote do: + result.add wrapBlock(quote do: let c_size = c_sizeof(`arg`) nim_size = sizeof(`arg`) @@ -47,18 +50,20 @@ macro testSizeAlignOf(args: varargs[untyped]): untyped = msg.add " align(get, expected): " & $nim_align & " != " & $c_align echo msg failed = true + ) macro testOffsetOf(a, b: untyped): untyped = let typeName = newLit(a.repr) let member = newLit(b.repr) - result = quote do: + result = wrapBlock(quote do: let c_offset = c_offsetof(`a`,`b`) nim_offset = offsetof(`a`,`b`) if c_offset != nim_offset: echo `typeName`, ".", `member`, " offsetError, C: ", c_offset, " nim: ", nim_offset failed = true + ) proc strAlign(arg: string): string = const minLen = 22 @@ -74,10 +79,11 @@ macro c_offsetof(fieldAccess: typed): int32 = else: fieldAccess let a = s[0].getTypeInst let b = s[1] - result = quote do: + result = wrapBlock(quote do: var res: int32 {.emit: [res, " = offsetof(", `a`, ", ", `b`, ");"] .} res + ) template c_offsetof(t: typedesc, a: untyped): int32 = var x: ptr t @@ -87,10 +93,11 @@ macro c_sizeof(a: typed): int32 = ## Bullet proof implementation that works using the sizeof operator ## in the c backend. Assuming of course this implementation is ## correct. - result = quote do: + result = wrapBlock(quote do: var res: int32 {.emit: [res, " = sizeof(", `a`, ");"] .} res + ) macro c_alignof(arg: untyped): untyped = ## Bullet proof implementation that works on actual alignment @@ -105,21 +112,23 @@ macro c_alignof(arg: untyped): untyped = macro testAlign(arg:untyped):untyped = let prefix = newLit(arg.lineinfo & " alignof " & arg.repr & " ") - result = quote do: + result = wrapBlock(quote do: let cAlign = c_alignof(`arg`) let nimAlign = alignof(`arg`) if cAlign != nimAlign: echo `prefix`, cAlign, " != ", nimAlign failed = true + ) macro testSize(arg:untyped):untyped = let prefix = newLit(arg.lineinfo & " sizeof " & arg.repr & " ") - result = quote do: + result = wrapBlock(quote do: let cSize = c_sizeof(`arg`) let nimSize = sizeof(`arg`) if cSize != nimSize: echo `prefix`, cSize, " != ", nimSize failed = true + ) type MyEnum {.pure.} = enum