First pass at integrating official Pug parser

DEVELOPMENT.md

@@ -18,7 +18,7 @@ ln -s "$(pwd)/flow/src/third-party/sedlex" src/sedlex
 ln -s "$(pwd)/flow/src/hack_forked/utils/collections" src/collections
 
 # JS dependencies
-npm install --no-save typescript browserify
+npm install --no-save typescript browserify pug-lexer pug-parser pug-walk
 
 # Install QuickJS
 curl https://bellard.org/quickjs/quickjs-2021-03-27.tar.xz > quickjs.tar.xz

Dockerfile

@@ -11,17 +11,20 @@ ENV DUNE_PROFILE release
 RUN opam update \
   && OPAMYES=1 opam install . --deps-only
 
+RUN echo '=== Installing QuickJS ===' \
+  && curl https://bellard.org/quickjs/quickjs-2021-03-27.tar.xz > quickjs.tar.xz \
+  && tar xvf quickjs.tar.xz && rm quickjs.tar.xz \
+  && mv quickjs-2021-03-27 quickjs \
+  && cd quickjs && make && cd -
+
 RUN echo '=== Installing Flow ===' \
   && git clone --branch v0.183.1 --depth 1 https://github.com/facebook/flow.git flow
 
 RUN echo '=== Installing TypeScript ===' \
   && npm install --no-save typescript browserify
 
-RUN echo '=== Installing QuickJS ===' \
-  && curl https://bellard.org/quickjs/quickjs-2021-03-27.tar.xz > quickjs.tar.xz \
-  && tar xvf quickjs.tar.xz && rm quickjs.tar.xz \
-  && mv quickjs-2021-03-27 quickjs \
-  && cd quickjs && make && cd -
+RUN echo '=== Installing Pug ===' \
+  && npm install --no-save pug-lexer pug-parser pug-walk
 
 COPY src src
 COPY dune dune

src/cli/strings.ml

@@ -63,7 +63,7 @@ let rec traverse ~root counts strings js_file_errors template_script directory =
         | { st_kind = S_REG; _ }, _, (lazy ".ts") ->
           process_file ~root strings counts.ts path ~f:(fun ic ->
               let* source = Lwt_io.read ic in
-              let+ parsed = Quickjs.extract_ts source in
+              let+ parsed = Quickjs.extract source Typescript in
               Array.iter parsed)
         | { st_kind = S_REG; _ }, (lazy ".vue"), _ ->
           process_file ~root strings counts.vue path ~f:(fun ic ->

src/cli/vue.ml

@@ -50,7 +50,7 @@ module Debug = struct
 end
 
 let extract_ts strings source =
-  let+ parsed = Quickjs.extract_ts source in
+  let+ parsed = Quickjs.extract source Typescript in
   Array.iter parsed ~f:(Queue.enqueue strings)
 
 let extract_template strings template_script possible_code =

src/quickjs/dune

@@ -22,10 +22,11 @@
 
 (rule
   (targets runtime.js)
+  (deps parsers.js)
   (action (bash "
     DIR=\"$(pwd)\"
     cd %{project_root}/../../
-    npx browserify src/quickjs/parser.js > \"$DIR/runtime.js\"
+    npx browserify src/quickjs/parsers.js > \"$DIR/runtime.js\"
   "))
   (mode standard)
 )

src/quickjs/parsers.js

@@ -0,0 +1,55 @@
+'use strict'
+
+const { createSourceFile, ScriptTarget, SyntaxKind, forEachChild } = require('typescript')
+
+globalThis.extractFromTypeScript = function (code) {
+  const acc = []
+  function traverse (node) {
+    if (node.kind === SyntaxKind.CallExpression
+      && node.expression.escapedText === 'L'
+      && node.arguments.length >= 1
+      && node.arguments[0].kind === SyntaxKind.StringLiteral
+    ) {
+      acc.push(node.arguments[0].text)
+    }
+    forEachChild(node, traverse)
+  }
+  traverse(createSourceFile('virtual.ts', code, ScriptTarget.ES2015, false))
+  return [acc, []]
+}
+
+const lex = require('pug-lexer')
+const parse = require('pug-parser')
+const walk = require('pug-walk')
+
+globalThis.extractFromPug = function (code) {
+  const ast = parse(lex(code))
+  const strings = []
+  const possibleJs = []
+  walk(ast, function before(node, replace) {
+    switch (node.type) {
+      case 'Tag':
+      case 'Filter':
+        node.attrs.forEach(({ val }) => possibleJs.push(val))
+        if (node.name === 'i18n') {
+          node.block.nodes.forEach(({ type, val }) => {
+            if (type === 'Text') {
+              strings.push(val)
+            }
+          })
+        }
+        break
+      case 'Code':
+        possibleJs.push(node.val)
+        break
+      case 'InterpolatedTag':
+        possibleJs.push(node.expr)
+        break
+      case 'Conditional':
+      case 'While':
+        possibleJs.push(node.test)
+        break
+    }
+  })
+  return [strings, possibleJs]
+}

src/quickjs/quickjs.cpp

@@ -54,13 +54,41 @@ value stub_init_contexts(value v_num_threads)
   CAMLreturn (v_ok_of_v(v_ret, Val_unit));
 }
 
+value convert_string_array(JSContext *ctx, JSValue jspair, int index, value v_ret, value v_field) {
+  CAMLparam2(v_ret, v_field);
+  JSValue jsarray = JS_GetPropertyUint32(ctx, jspair, index);
+  JSValue prop_len = JS_GetPropertyStr(ctx, jsarray, "length");
+  int64_t len;
+  if (JS_ToInt64(ctx, &len, prop_len)) {
+    JS_FreeValue(ctx, prop_len);
+    JS_FreeValue(ctx, jsarray);
+    CAMLreturn (v_error_of_cstring(v_ret, v_field, "Could not get results length, please report this bug."));
+  }
+
+  v_field = caml_alloc(len, 0);
+  for (int i = 0; i < len; i++) {
+    JSValue prop_str = JS_GetPropertyUint32(ctx, jsarray, i);
+    size_t str_len;
+    const char* str = JS_ToCStringLen(ctx, &str_len, prop_str);
+    Store_field(v_field, i, caml_alloc_initialized_string(str_len, str));
+    JS_FreeCString(ctx, str);
+    JS_FreeValue(ctx, prop_str);
+  }
+
+  JS_FreeValue(ctx, prop_len);
+  JS_FreeValue(ctx, jsarray);
+
+  CAMLreturn (v_ok_of_v(v_ret, v_field));
+}
+
 extern "C"
-value stub_extract_ts(value v_id, value v_code)
+value stub_extract(value v_id, value v_code, value v_fn_name)
 {
-  CAMLparam2(v_id, v_code);
-  CAMLlocal2(v_ret, v_field);
+  CAMLparam3(v_id, v_code, v_fn_name);
+  CAMLlocal4(v_ret, v_field, v_arr1, v_arr2);
   int id = Int_val(v_id);
   string code(String_val(v_code), caml_string_length(v_code));
+  string fn_name(String_val(v_fn_name), caml_string_length(v_fn_name));
 
   caml_enter_blocking_section();
 
@@ -73,7 +101,7 @@ value stub_extract_ts(value v_id, value v_code)
 
   JSValue code_val = JS_NewStringLen(ctx, code.c_str(), code.length());
   JSValue global_obj = JS_GetGlobalObject(ctx);
-  JSValue fun = JS_GetPropertyStr(ctx, global_obj, "extract");
+  JSValue fun = JS_GetPropertyStr(ctx, global_obj, fn_name.c_str());
   JSValue ret_val = JS_Call(ctx, fun, JS_NULL, 1, &code_val);
   JS_FreeValue(ctx, fun);
   JS_FreeValue(ctx, global_obj);
@@ -87,29 +115,27 @@ value stub_extract_ts(value v_id, value v_code)
     CAMLreturn (v_ret);
   };
 
-  JSValue prop_len = JS_GetPropertyStr(ctx, ret_val, "length");
-  int64_t len;
-  if (JS_ToInt64(ctx, &len, prop_len)) {
-    JS_FreeValue(ctx, prop_len);
-    JS_FreeValue(ctx, ret_val);
-    caml_leave_blocking_section();
-    CAMLreturn (v_error_of_cstring(v_ret, v_field, "Could not get TS results length, please report this bug."));
-  }
-  JS_FreeValue(ctx, prop_len);
-
   caml_leave_blocking_section();
 
-  v_field = caml_alloc(len, 0);
-  for (int i = 0; i < len; i++) {
-    JSValue prop_str = JS_GetPropertyUint32(ctx, ret_val, i);
-    size_t str_len;
-    const char* str = JS_ToCStringLen(ctx, &str_len, prop_str);
-    Store_field(v_field, i, caml_alloc_initialized_string(str_len, str));
-    JS_FreeCString(ctx, str);
-    JS_FreeValue(ctx, prop_str);
+  v_arr1 = convert_string_array(ctx, ret_val, 0, v_ret, v_field);
+  v_arr2 = convert_string_array(ctx, ret_val, 1, v_ret, v_field);
+  if (Tag_val(v_arr1) == 0 && Tag_val(v_arr2) == 0) {
+    // Ok, Ok
+    // Tuple2
+    v_field = caml_alloc_small(2, 0);
+    Field(v_field, 0) = Field(v_arr1, 0);
+    Field(v_field, 1) = Field(v_arr2, 0);
+    v_ret = v_ok_of_v(v_ret, v_field);
+
+  } else if (Tag_val(v_arr1) == 1) {
+    // Error, _
+    v_ret = v_arr1;
+  } else if (Tag_val(v_arr2) == 1) {
+    // Ok, Error
+    v_ret = v_arr2;
   }
 
   JS_FreeValue(ctx, ret_val);
 
-  CAMLreturn (v_ok_of_v(v_ret, v_field));
+  CAMLreturn (v_ret);
 }

src/quickjs/quickjs.ml

@@ -2,9 +2,23 @@ open Core
 open Lwt.Infix
 open Lwt.Syntax
 
+type pug_response = {
+  strings: string array;
+  possible_js: string array;
+}
+
+type _ kind =
+  | Typescript : string array kind
+  | Pug : pug_response kind
+
+let fn_name_of_kind (type a) : a kind -> string = function
+| Typescript -> "extractFromTypeScript"
+| Pug -> "extractFromPug"
+
 external stub_init_contexts : int -> (unit, string) Result.t = "stub_init_contexts"
 
-external stub_extract_ts : int -> string -> (string array, string) Result.t = "stub_extract_ts"
+external stub_extract : int -> string -> fn_name:string -> (string array * string array, string) Result.t
+  = "stub_extract"
 
 let num_threads = 4
 
@@ -27,7 +41,14 @@ let js_contexts =
       incr ctr;
       Lwt.return i)
 
-let extract_ts code =
+let extract (type a) code (kind : a kind) : a Lwt.t =
   let* () = force init_contexts in
-  Lwt_pool.use js_contexts (fun id -> Lwt_preemptive.detach (fun () -> stub_extract_ts id code) ())
-  >|= Result.ok_or_failwith
+  let fn_name = fn_name_of_kind kind in
+  Lwt_pool.use js_contexts (fun id -> Lwt_preemptive.detach (fun () -> stub_extract id code ~fn_name) ())
+  >|= function
+  | Error msg -> failwith msg
+  | Ok (strings, possible_js) ->
+    (match kind with
+     | Typescript -> strings
+     | Pug -> { strings; possible_js }
+      : a)