Skip to content

Commit

Permalink
Refactoring, groundwork for official pug parser
Browse files Browse the repository at this point in the history
  • Loading branch information
@SGrondin
SGrondin committed Sep 11, 2022
1 parent 04021e7 commit bd39903
Show file tree
Hide file tree
Showing 14 changed files with 295 additions and 273 deletions.
115 changes: 61 additions & 54 deletions src/cli/strings.ml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,17 +32,26 @@ type counts = {
ts: int ref;
}

let process_file ~root strings count filename ~f:get_iter =
let process_file ~root table count filename template_script ~f:get_collector : unit Lwt.t =
Lwt_pool.use pool (fun () ->
incr count;
let+ iter = Lwt_io.with_file ~mode:Input ~flags:read_flags filename get_iter in
iter ~f:(fun string ->
let data = String.chop_prefix filename ~prefix:root |> Option.value ~default:filename in
String.Table.update strings string ~f:(function
| None -> String.Set.add String.Set.empty data
| Some set -> String.Set.add set data)))
let* (collector : Utils.Collector.t) =
Lwt_io.with_file ~mode:Input ~flags:read_flags filename get_collector
in
let handler string =
let data = String.chop_prefix filename ~prefix:root |> Option.value ~default:filename in
String.Table.update table string ~f:(function
| None -> String.Set.add String.Set.empty data
| Some set -> String.Set.add set data)
in
let* () =
Utils.Collector.render_errors collector
|> Option.value_map ~default:Lwt.return_unit ~f:(Lwt_io.eprintlf "❌ %s")
in
Queue.iter collector.strings ~f:handler;
Vue.collect_from_possible_scripts collector template_script ~on_string:handler)

let rec traverse ~root counts strings js_file_errors template_script directory =
let rec traverse ~root counts strings template_script directory =
let* entries =
Lwt_pool.use pool (fun () -> Lwt_unix.files_of_directory directory |> Lwt_stream.to_list)
in
Expand All @@ -55,38 +64,40 @@ let rec traverse ~root counts strings js_file_errors template_script directory =
Lwt_unix.lstat path >>= fun stat ->
match stat, lazy (String.slice filename (-4) 0), lazy (String.slice filename (-3) 0) with
| { st_kind = S_REG; _ }, _, (lazy ".js") when String.is_suffix ~suffix:".js" filename ->
process_file ~root strings counts.js path ~f:(fun ic ->
let* source = Lwt_io.read ic in
let parsed = Queue.create () in
let+ () = Parsing.Js_ast.strings_from_js ~filename:path parsed js_file_errors source in
Queue.iter parsed)
process_file ~root strings counts.js path template_script ~f:(fun ic ->
let collector = Utils.Collector.create ~filename in
let+ source = Lwt_io.read ic in
Parsing.Js.extract_to_collector collector source;
collector)
| { st_kind = S_REG; _ }, _, (lazy ".ts") ->
process_file ~root strings counts.ts path ~f:(fun ic ->
process_file ~root strings counts.ts path template_script ~f:(fun ic ->
let collector = Utils.Collector.create ~filename in
let* source = Lwt_io.read ic in
let+ parsed = Quickjs.extract source Typescript in
Array.iter parsed)
let+ () = Quickjs.extract_to_collector collector Typescript source in
collector)
| { st_kind = S_REG; _ }, (lazy ".vue"), _ ->
process_file ~root strings counts.vue path ~f:(fun ic ->
process_file ~root strings counts.vue path template_script ~f:(fun ic ->
let collector = Utils.Collector.create ~filename in
let* languages = Vue.parse ~filename ic in
Vue.extract_strings ~filename js_file_errors template_script languages)
let+ () = Vue.collect_from_languages collector languages in
collector)
| { st_kind = S_REG; _ }, (lazy ".pug"), _ ->
process_file ~root strings counts.pug path ~f:(fun ic ->
let* nodes =
process_file ~root strings counts.pug path template_script ~f:(fun ic ->
let collector = Utils.Collector.create ~filename in
let+ parsed =
Parsing.Basic.exec_parser_lwt Parsing.Pug.parser ~filename ~language_name:"Pug" ic
in
let parsed = Queue.create () in
let+ () = Vue.collect_pug parsed template_script nodes in
Queue.iter parsed)
Parsing.Pug.collect collector parsed;
collector)
| { st_kind = S_REG; _ }, _, _ when String.is_suffix filename ~suffix:".html" ->
process_file ~root strings counts.html path ~f:(fun ic ->
let* top =
process_file ~root strings counts.html path template_script ~f:(fun ic ->
let collector = Utils.Collector.create ~filename in
let+ parsed =
Parsing.Basic.exec_parser_lwt Parsing.Html.parser ~filename ~language_name:"HTML" ic
in
let parsed = Queue.create () in
let+ () = Vue.collect_html parsed template_script top in
Queue.iter parsed)
| { st_kind = S_DIR; _ }, _, _ ->
traverse ~root counts strings js_file_errors template_script path
Parsing.Html.collect collector parsed;
collector)
| { st_kind = S_DIR; _ }, _, _ -> traverse ~root counts strings template_script path
| _ -> Lwt.return_unit))
entries

Expand Down Expand Up @@ -197,6 +208,7 @@ let directory_exists path =
type common_options = {
targets: string list;
template_script: Vue.template_script;
fast_pug: bool;
}

let handle_system_failure = function
Expand All @@ -212,7 +224,7 @@ type action =
| Debug of Vue.Debug.t
| Run

let main { targets; template_script } = function
let main { targets; template_script; fast_pug } = function
| Debug lang ->
Lwt_list.iter_s
(fun filename ->
Expand All @@ -222,16 +234,21 @@ let main { targets; template_script } = function
| _, ".vue" ->
let* languages = Vue.parse ~filename ic in
Vue.debug_template ~filename languages template_script lang
| Pug, ".pug" ->
let* nodes =
| Pug, ".pug" when fast_pug ->
let* parsed =
Parsing.Basic.exec_parser_lwt Parsing.Pug.parser ~filename ~language_name:"Pug" ic
in
Vue.debug_template ~filename [ Pug { nodes; length = None } ] template_script lang
Vue.debug_template ~filename [ Pug_native { parsed; length = None } ] template_script lang
| Pug, ".pug" ->
let* source = Lwt_io.read ic in
let* xx = Quickjs.extract Pug source in

Lwt.return_unit
| Html, _ when String.is_suffix filename ~suffix:".html" ->
let* top =
let* parsed =
Parsing.Basic.exec_parser_lwt Parsing.Html.parser ~filename ~language_name:"Pug" ic
in
Vue.debug_template ~filename [ Html { top; length = None } ] template_script lang
Vue.debug_template ~filename [ Html { parsed; length = None } ] template_script lang
| _ -> Lwt_io.printlf "Nothing to do for file [%s]" filename))
targets
| Run ->
Expand All @@ -250,7 +267,6 @@ let main { targets; template_script } = function
[]
in
(* English *)
let js_file_errors = Queue.create () in
let* english =
let english_list = String.Table.create () in
let counts = { vue = ref 0; pug = ref 0; html = ref 0; js = ref 0; ts = ref 0 } in
Expand All @@ -259,7 +275,7 @@ let main { targets; template_script } = function
Lwt_list.iter_p
(fun directory ->
let root = String.chop_suffix ~suffix:"/" directory |> Option.value ~default:directory in
traverse ~root:(sprintf "%s/" root) counts english_list js_file_errors template_script root)
traverse ~root:(sprintf "%s/" root) counts english_list template_script root)
targets
in
let english =
Expand Down Expand Up @@ -295,28 +311,19 @@ let main { targets; template_script } = function
| None -> Lwt.return_unit)
strings_dir_files
in
(* JS parse errors *)
let* () =
match Queue.length js_file_errors with
| 0 -> Lwt.return_unit
| len ->
let files =
Queue.to_array js_file_errors
|> Array.map ~f:(fun { filename; _ } -> sprintf "- %s" filename)
|> String.concat_array ~sep:"\n"
in
Lwt_io.printlf "❌ Encountered %d parsing error%s. File%s:\n%s\n" len (plural len) (plural len) files
in
Lwt_io.write_line Lwt_io.stdout (sprintf !"Completed. (%{Int63}ms)" (overall_time ()))

let () =
let open Command in
let open Command.Let_syntax in
let common =
let%map_open use_ts =
flag "--ts" ~full_flag_required:() no_arg ~doc:"Interpret Vue templates as TypeScript"
and targets = Param.("path" %: string |> sequence |> anon) in
{ targets; template_script = (if use_ts then TS else JS) }
let%map_open targets = Param.("path" %: string |> sequence |> anon)
and use_ts = flag "--ts" ~full_flag_required:() no_arg ~doc:"Interpret Vue templates as TypeScript"
and fast_pug =
flag "--fast-pug" ~full_flag_required:() no_arg
~doc:"Use the native Pug parser. Much faster but not every Pug feature is supported."
in
{ targets; template_script = (if use_ts then TS else JS); fast_pug }
in
let action =
let open Param in
Expand Down
170 changes: 67 additions & 103 deletions src/cli/vue.ml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,25 +12,25 @@ end

module Language = struct
type t =
| Js of string
| Ts of string
| Html of {
top: SZXX.Xml.DOM.element;
| Js of string
| Ts of string
| Html of {
parsed: Html.t;
length: int option;
}
| Pug of {
nodes: Pug.node array;
| Pug_native of {
parsed: Pug.t;
length: int option;
}
| Css of int
| Css of int

let of_source ~filename : Source.t -> t = function
| Template (Template.HTML source) ->
let top = Parsing.Basic.exec_parser Parsing.Html.parser ~filename ~language_name:"HTML" source in
Html { top; length = Some (String.length source) }
let parsed = Parsing.Basic.exec_parser Parsing.Html.parser ~filename ~language_name:"HTML" source in
Html { parsed; length = Some (String.length source) }
| Template (Template.PUG source) ->
let nodes = Basic.exec_parser Pug.parser ~filename ~language_name:"Pug" source in
Pug { nodes; length = Some (String.length source) }
let parsed = Basic.exec_parser Pug.parser ~filename ~language_name:"Pug" source in
Pug_native { parsed; length = Some (String.length source) }
| Script (Script.JS s) -> Js s
| Script (Script.TS s) -> Ts s
| Style (Style.CSS s) -> Css (String.length s)
Expand All @@ -46,104 +46,68 @@ module Debug = struct
| Html
end

let extract_ts strings source =
let+ parsed = Quickjs.extract source Typescript in
Array.iter parsed ~f:(Queue.enqueue strings)
let collect_from_possible_scripts Utils.Collector.{ possible_scripts; _ } template_script ~on_string =
Queue.fold possible_scripts ~init:Lwt.return_unit ~f:(fun acc raw ->
let* () = acc in
match template_script with
| JS ->
Js.extract raw ~on_string;
Lwt.return_unit
| TS -> (
Quickjs.extract Typescript raw >|= function
| Error _ -> ()
| Ok (strings, _) -> Array.iter strings ~f:on_string))

let extract_template strings template_script possible_code =
match template_script with
| JS ->
List.iter possible_code ~f:(Js_ast.strings_from_template strings);
Lwt.return_unit
| TS -> Lwt_list.iter_p (extract_ts strings) possible_code

let collect_pug strings template_script nodes =
let rec loop acc Pug.{ selector; arguments; text; children } =
let acc =
match text, selector with
| Some s, Element { parts = "i18n" :: _ } ->
Queue.enqueue strings s;
acc
| Some s, _ -> s :: acc
| None, _ -> acc
in
let acc =
List.fold arguments ~init:acc ~f:(fun acc -> function
| Pug.{ contents = None; _ } -> acc
| Pug.{ contents = Some s; _ } -> s :: acc)
in
Array.fold children ~init:acc ~f:loop
in
Array.fold nodes ~init:[] ~f:loop |> extract_template strings template_script

let collect_html strings template_script top =
let rec loop acc (node : SZXX.Xml.DOM.element) =
let acc =
match node with
| { text = ""; _ } -> acc
| { tag = "i18n"; text; _ } ->
Queue.enqueue strings text;
acc
| { text; _ } -> text :: acc
in
let acc =
List.fold node.attrs ~init:acc ~f:(fun acc -> function
| "class", _
|"id", _
|_, "" ->
acc
| _, source -> SZXX.Xml.unescape source :: acc)
in
Array.fold node.children ~init:acc ~f:loop
in
loop [] top |> extract_template strings template_script

let extract_strings ~filename js_file_errors template_script languages =
let strings = Queue.create () in
let+ () =
Lwt_list.iter_p
(function
| Language.Html { top; length = _ } -> collect_html strings template_script top
| Pug { nodes; length = _ } -> collect_pug strings template_script nodes
| Js source -> Js_ast.strings_from_js ~filename strings js_file_errors source
| Ts source -> extract_ts strings source
| Css _ -> Lwt.return_unit)
languages
in
Queue.iter strings
let collect_from_languages collector languages =
Lwt_list.iter_p
(function
| Language.Html { parsed; length = _ } ->
Html.collect collector parsed;
Lwt.return_unit
| Pug_native { parsed; length = _ } ->
Pug.collect collector parsed;
Lwt.return_unit
| Js source ->
Js.extract_to_collector collector source;
Lwt.return_unit
| Ts source -> Quickjs.extract_to_collector collector Typescript source
| Css _ -> Lwt.return_unit)
languages

let debug_template ~filename languages template_script target =
let print_iter iter =
let print_collector ~error_kind (Utils.Collector.{ strings; file_errors; _ } as collector) =
let* () =
collect_from_possible_scripts collector template_script ~on_string:(Queue.enqueue strings)
in
let buf = Buffer.create 256 in
iter ~f:(fun s ->
Buffer.add_string buf s;
Buffer.add_char buf '\n');
Queue.iter strings ~f:(fun s -> bprintf buf "%s\n" s);
if not (Queue.is_empty file_errors)
then (
bprintf buf "\n%s errors in %s:\n" error_kind filename;
Queue.iter file_errors ~f:(bprintf buf "- %s\n"));
Lwt_io.printl (Buffer.contents buf)
in
let js_file_errors = Queue.create () in
let* () =
Lwt_list.iter_s
(fun lang ->
match (lang : Language.t), (target : Debug.t) with
| Js source, _ -> Lwt_io.printlf "<JS Code - %d bytes>" (String.length source)
| Ts source, _ -> Lwt_io.printlf "<TS Code - %d bytes>" (String.length source)
| Css length, _ -> Lwt_io.printlf "<CSS Code - %d bytes>" length
| Html { top; length = _ }, Html ->
let* () = Lwt_io.printlf !"%{sexp#hum: SZXX.Xml.DOM.element}" top in
let* iter = extract_strings ~filename js_file_errors template_script [ lang ] in
print_iter iter
| (Pug { nodes; length = _ } as lang), Pug ->
let* () = Lwt_io.printlf !"%{sexp#hum: Pug.t}" nodes in
let* iter = extract_strings ~filename js_file_errors template_script [ lang ] in
print_iter iter
| Html { length = Some len; _ }, Pug -> Lwt_io.printlf "<HTML code - %d bytes>" len
| Html { length = None; _ }, Pug -> Lwt_io.printl "<HTML code>"
| Pug { length = Some len; _ }, Html -> Lwt_io.printlf "<Pug code - %d bytes>" len
| Pug { length = None; _ }, Html -> Lwt_io.printl "<Pug code>")
languages
in
Lwt_io.printl
(Queue.to_array js_file_errors |> Array.map ~f:Utils.Failed.to_string |> String.concat_array ~sep:"\n")
Lwt_list.iter_s
(fun lang ->
match (lang : Language.t), (target : Debug.t) with
| Js source, _ -> Lwt_io.printlf "<JS Code - %d bytes>" (String.length source)
| Ts source, _ -> Lwt_io.printlf "<TS Code - %d bytes>" (String.length source)
| Css length, _ -> Lwt_io.printlf "<CSS Code - %d bytes>" length
| Html { parsed; length = _ }, Html ->
let collector = Utils.Collector.create ~filename in
let* () = Lwt_io.printlf !"%{sexp#hum: Html.t}" parsed in
let* () = collect_from_languages collector [ lang ] in
print_collector ~error_kind:"HTML" collector
| (Pug_native { parsed; length = _ } as lang), Pug ->
let* () = Lwt_io.printlf !"%{sexp#hum: Pug.t}" parsed in
let collector = Utils.Collector.create ~filename in
let* () = collect_from_languages collector [ lang ] in
print_collector ~error_kind:"Pug" collector
| Html { length = Some len; _ }, Pug -> Lwt_io.printlf "<HTML code - %d bytes>" len
| Html { length = None; _ }, Pug -> Lwt_io.printl "<HTML code>"
| Pug_native { length = Some len; _ }, Html -> Lwt_io.printlf "<Pug code - %d bytes>" len
| Pug_native { length = None; _ }, Html -> Lwt_io.printl "<Pug code>")
languages

let parse ~filename ic =
let open Angstrom in
Expand Down
Loading

0 comments on commit bd39903

Please sign in to comment.