Skip to content

Commit

Permalink
Fully wired in official pug parser
Browse files Browse the repository at this point in the history
  • Loading branch information
@SGrondin
SGrondin committed Sep 14, 2022
1 parent bd39903 commit 788dbaf
Show file tree
Hide file tree
Showing 9 changed files with 190 additions and 110 deletions.
82 changes: 42 additions & 40 deletions src/cli/strings.ml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,16 @@ type counts = {
ts: int ref;
}

type common_options = {
targets: string list;
template_script: Vue.template_script;
fast_pug: bool;
}

type action =
| Debug of Vue.Debug.t
| Run

let process_file ~root table count filename template_script ~f:get_collector : unit Lwt.t =
Lwt_pool.use pool (fun () ->
incr count;
Expand All @@ -46,12 +56,12 @@ let process_file ~root table count filename template_script ~f:get_collector : u
in
let* () =
Utils.Collector.render_errors collector
|> Option.value_map ~default:Lwt.return_unit ~f:(Lwt_io.eprintlf "❌ %s")
|> Option.value_map ~default:Lwt.return_unit ~f:Lwt_io.eprintl
in
Queue.iter collector.strings ~f:handler;
Vue.collect_from_possible_scripts collector template_script ~on_string:handler)

let rec traverse ~root counts strings template_script directory =
let rec traverse ~root counts strings ({ template_script; fast_pug; _ } as options) directory =
let* entries =
Lwt_pool.use pool (fun () -> Lwt_unix.files_of_directory directory |> Lwt_stream.to_list)
in
Expand All @@ -65,39 +75,39 @@ let rec traverse ~root counts strings template_script directory =
match stat, lazy (String.slice filename (-4) 0), lazy (String.slice filename (-3) 0) with
| { st_kind = S_REG; _ }, _, (lazy ".js") when String.is_suffix ~suffix:".js" filename ->
process_file ~root strings counts.js path template_script ~f:(fun ic ->
let collector = Utils.Collector.create ~filename in
let collector = Utils.Collector.create ~path in
let+ source = Lwt_io.read ic in
Parsing.Js.extract_to_collector collector source;
collector)
| { st_kind = S_REG; _ }, _, (lazy ".ts") ->
process_file ~root strings counts.ts path template_script ~f:(fun ic ->
let collector = Utils.Collector.create ~filename in
let collector = Utils.Collector.create ~path in
let* source = Lwt_io.read ic in
let+ () = Quickjs.extract_to_collector collector Typescript source in
collector)
| { st_kind = S_REG; _ }, (lazy ".vue"), _ ->
process_file ~root strings counts.vue path template_script ~f:(fun ic ->
let collector = Utils.Collector.create ~filename in
let* languages = Vue.parse ~filename ic in
let collector = Utils.Collector.create ~path in
let* languages = Vue.parse ~path ~fast_pug ic in
let+ () = Vue.collect_from_languages collector languages in
collector)
| { st_kind = S_REG; _ }, (lazy ".pug"), _ ->
process_file ~root strings counts.pug path template_script ~f:(fun ic ->
let collector = Utils.Collector.create ~filename in
let collector = Utils.Collector.create ~path in
let+ parsed =
Parsing.Basic.exec_parser_lwt Parsing.Pug.parser ~filename ~language_name:"Pug" ic
Parsing.Basic.exec_parser_lwt Parsing.Pug.parser ~path ~language_name:"Pug" ic
in
Parsing.Pug.collect collector parsed;
collector)
| { st_kind = S_REG; _ }, _, _ when String.is_suffix filename ~suffix:".html" ->
process_file ~root strings counts.html path template_script ~f:(fun ic ->
let collector = Utils.Collector.create ~filename in
let collector = Utils.Collector.create ~path in
let+ parsed =
Parsing.Basic.exec_parser_lwt Parsing.Html.parser ~filename ~language_name:"HTML" ic
Parsing.Basic.exec_parser_lwt Parsing.Html.parser ~path ~language_name:"HTML" ic
in
Parsing.Html.collect collector parsed;
collector)
| { st_kind = S_DIR; _ }, _, _ -> traverse ~root counts strings template_script path
| { st_kind = S_DIR; _ }, _, _ -> traverse ~root counts strings options path
| _ -> Lwt.return_unit))
entries

Expand Down Expand Up @@ -205,12 +215,6 @@ let directory_exists path =
| { st_kind = _; _ } -> failwithf "%s already exists, but is not a directory" path ()
| exception _ -> false

type common_options = {
targets: string list;
template_script: Vue.template_script;
fast_pug: bool;
}

let handle_system_failure = function
| (Failure _ as ex)
|(Core_unix.Unix_error _ as ex)
Expand All @@ -220,36 +224,34 @@ let handle_system_failure = function
exit 1
| exn -> raise exn

type action =
| Debug of Vue.Debug.t
| Run

let main { targets; template_script; fast_pug } = function
let main ({ targets; template_script; fast_pug } as options) = function
| Debug lang ->
Lwt_list.iter_s
(fun filename ->
let* () = Lwt_io.printlf "Debugging [%s]" filename in
Lwt_io.with_file ~flags:read_flags ~mode:Input filename (fun ic ->
match lang, String.slice filename (-4) 0 with
(fun path ->
let* () = Lwt_io.printlf "\n>>> Debugging [%s]" path in
Lwt_io.with_file ~flags:read_flags ~mode:Input path (fun ic ->
match lang, String.slice path (-4) 0 with
| _, ".vue" ->
let* languages = Vue.parse ~filename ic in
Vue.debug_template ~filename languages template_script lang
let* languages = Vue.parse ~path ~fast_pug ic in
Vue.debug_template ~path languages template_script lang
| Pug, ".pug" when fast_pug ->
let* parsed =
Parsing.Basic.exec_parser_lwt Parsing.Pug.parser ~filename ~language_name:"Pug" ic
Parsing.Basic.exec_parser_lwt Parsing.Pug.parser ~path ~language_name:"Pug" ic
in
Vue.debug_template ~filename [ Pug_native { parsed; length = None } ] template_script lang
Vue.debug_template ~path [ Pug_native { parsed; length = None } ] template_script lang
| Pug, ".pug" ->
let* source = Lwt_io.read ic in
let* xx = Quickjs.extract Pug source in

Lwt.return_unit
| Html, _ when String.is_suffix filename ~suffix:".html" ->
let collector = Utils.Collector.create ~path in
let* () = Quickjs.extract_to_collector collector Pug source in
Vue.debug_template ~path
[ Pug { collector; length = String.length source } ]
template_script lang
| Html, _ when String.is_suffix path ~suffix:".html" ->
let* parsed =
Parsing.Basic.exec_parser_lwt Parsing.Html.parser ~filename ~language_name:"Pug" ic
Parsing.Basic.exec_parser_lwt Parsing.Html.parser ~path ~language_name:"Pug" ic
in
Vue.debug_template ~filename [ Html { parsed; length = None } ] template_script lang
| _ -> Lwt_io.printlf "Nothing to do for file [%s]" filename))
Vue.debug_template ~path [ Html { parsed; length = None } ] template_script lang
| _ -> Lwt_io.printlf "Nothing to do for file [%s]" path))
targets
| Run ->
let overall_time = Utils.time () in
Expand All @@ -275,7 +277,7 @@ let main { targets; template_script; fast_pug } = function
Lwt_list.iter_p
(fun directory ->
let root = String.chop_suffix ~suffix:"/" directory |> Option.value ~default:directory in
traverse ~root:(sprintf "%s/" root) counts english_list template_script root)
traverse ~root:(sprintf "%s/" root) counts english_list options root)
targets
in
let english =
Expand Down Expand Up @@ -304,7 +306,7 @@ let main { targets; template_script; fast_pug } = function
Lwt_unix.stat path >>= function
| { st_kind = S_REG; _ } ->
let* other =
Lwt_io.with_file ~mode:Input ~flags:read_flags path (Parsing.Strings.parse ~filename)
Lwt_io.with_file ~mode:Input ~flags:read_flags path (Parsing.Strings.parse ~path)
in
write_other ~language english other
| _ -> Lwt.return_unit)
Expand All @@ -320,7 +322,7 @@ let () =
let%map_open targets = Param.("path" %: string |> sequence |> anon)
and use_ts = flag "--ts" ~full_flag_required:() no_arg ~doc:"Interpret Vue templates as TypeScript"
and fast_pug =
flag "--fast-pug" ~full_flag_required:() no_arg
flag "--fast-pug" ~aliases:[ "--fp" ] ~full_flag_required:() no_arg
~doc:"Use the native Pug parser. Much faster but not every Pug feature is supported."
in
{ targets; template_script = (if use_ts then TS else JS); fast_pug }
Expand Down
67 changes: 41 additions & 26 deletions src/cli/vue.ml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,18 +22,26 @@ module Language = struct
parsed: Pug.t;
length: int option;
}
| Pug of {
collector: Utils.Collector.t;
length: int;
}
| Css of int

let of_source ~filename : Source.t -> t = function
let of_source ~path ~fast_pug : Source.t -> t Lwt.t = function
| Template (Template.HTML source) ->
let parsed = Parsing.Basic.exec_parser Parsing.Html.parser ~filename ~language_name:"HTML" source in
Html { parsed; length = Some (String.length source) }
let parsed = Parsing.Basic.exec_parser Parsing.Html.parser ~path ~language_name:"HTML" source in
Html { parsed; length = Some (String.length source) } |> Lwt.return
| Template (Template.PUG source) when fast_pug ->
let parsed = Basic.exec_parser Pug.parser ~path ~language_name:"Pug" source in
Pug_native { parsed; length = Some (String.length source) } |> Lwt.return
| Template (Template.PUG source) ->
let parsed = Basic.exec_parser Pug.parser ~filename ~language_name:"Pug" source in
Pug_native { parsed; length = Some (String.length source) }
| Script (Script.JS s) -> Js s
| Script (Script.TS s) -> Ts s
| Style (Style.CSS s) -> Css (String.length s)
let collector = Utils.Collector.create ~path in
let+ () = Quickjs.extract_to_collector collector Pug source in
Pug { collector; length = String.length source }
| Script (Script.JS s) -> Js s |> Lwt.return
| Script (Script.TS s) -> Ts s |> Lwt.return
| Style (Style.CSS s) -> Css (String.length s) |> Lwt.return
end

type template_script =
Expand Down Expand Up @@ -67,14 +75,17 @@ let collect_from_languages collector languages =
| Pug_native { parsed; length = _ } ->
Pug.collect collector parsed;
Lwt.return_unit
| Pug { collector = src; length = _ } ->
Utils.Collector.blit_transfer ~src ~dst:collector;
Lwt.return_unit
| Js source ->
Js.extract_to_collector collector source;
Lwt.return_unit
| Ts source -> Quickjs.extract_to_collector collector Typescript source
| Css _ -> Lwt.return_unit)
languages

let debug_template ~filename languages template_script target =
let debug_template ~path languages template_script target =
let print_collector ~error_kind (Utils.Collector.{ strings; file_errors; _ } as collector) =
let* () =
collect_from_possible_scripts collector template_script ~on_string:(Queue.enqueue strings)
Expand All @@ -83,7 +94,7 @@ let debug_template ~filename languages template_script target =
Queue.iter strings ~f:(fun s -> bprintf buf "%s\n" s);
if not (Queue.is_empty file_errors)
then (
bprintf buf "\n%s errors in %s:\n" error_kind filename;
bprintf buf "\n❌ %s errors in %s:\n" error_kind path;
Queue.iter file_errors ~f:(bprintf buf "- %s\n"));
Lwt_io.printl (Buffer.contents buf)
in
Expand All @@ -94,33 +105,37 @@ let debug_template ~filename languages template_script target =
| Ts source, _ -> Lwt_io.printlf "<TS Code - %d bytes>" (String.length source)
| Css length, _ -> Lwt_io.printlf "<CSS Code - %d bytes>" length
| Html { parsed; length = _ }, Html ->
let collector = Utils.Collector.create ~filename in
let collector = Utils.Collector.create ~path in
let* () = Lwt_io.printlf !"%{sexp#hum: Html.t}" parsed in
let* () = collect_from_languages collector [ lang ] in
print_collector ~error_kind:"HTML" collector
| (Pug_native { parsed; length = _ } as lang), Pug ->
let* () = Lwt_io.printlf !"%{sexp#hum: Pug.t}" parsed in
let collector = Utils.Collector.create ~filename in
let collector = Utils.Collector.create ~path in
let* () = collect_from_languages collector [ lang ] in
print_collector ~error_kind:"Pug" collector
| Pug { collector; length = _ }, Pug -> print_collector ~error_kind:"Pug" collector
| Html { length = Some len; _ }, Pug -> Lwt_io.printlf "<HTML code - %d bytes>" len
| Html { length = None; _ }, Pug -> Lwt_io.printl "<HTML code>"
| Pug_native { length = Some len; _ }, Html -> Lwt_io.printlf "<Pug code - %d bytes>" len
| Pug_native { length = None; _ }, Html -> Lwt_io.printl "<Pug code>")
| Pug_native { length = None; _ }, Html -> Lwt_io.printl "<Pug code>"
| Pug { length; _ }, Html -> Lwt_io.printlf "<Pug code - %d bytes>" length)
languages

let parse ~filename ic =
let open Angstrom in
let open Basic in
let parse ~path ~fast_pug ic =
let buf = Buffer.create 256 in
let languages =
choice
[
(Template.parser buf >>| fun x -> Source.Template x);
(Script.parser buf >>| fun x -> Source.Script x);
(Style.parser buf >>| fun x -> Source.Style x);
]
let parser =
let open Angstrom in
let open Basic in
let languages =
choice
[
(Template.parser buf >>| fun x -> Source.Template x);
(Script.parser buf >>| fun x -> Source.Script x);
(Style.parser buf >>| fun x -> Source.Style x);
]
in
mlws *> sep_by mlws languages <* mlws
in
let parser = mlws *> sep_by mlws languages <* mlws in
Basic.exec_parser_lwt parser ~filename ~language_name:"Vue" ic
>|= List.map ~f:(Language.of_source ~filename)
Basic.exec_parser_lwt parser ~path ~language_name:"Vue" ic
>>= Lwt_list.map_p (Language.of_source ~path ~fast_pug)
18 changes: 9 additions & 9 deletions src/parsing/basic.ml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,32 +108,32 @@ let block_parser (starts, ends) buf ~f =
<* many_till line ends
>>| fun x -> f (Buffer.contents buf) x

let default_error_message ~filename ~language_name ~unparsed =
let default_error_message ~path ~language_name ~unparsed =
sprintf
"The file [%s] contains invalid syntax or %s features unsupported by this tool.\n\
Please report this so it can be improved.\n\
The unsupported syntax starts at:\n\
%s"
filename language_name
path language_name
(Yojson.Basic.to_string (`String (String.slice unparsed 0 Int.(min 20 (String.length unparsed)))))

let default_syntax_error ~filename ~language_name ~err =
let default_syntax_error ~path ~language_name ~err =
failwithf
"The file [%s] contains invalid syntax or %s features unsupported by this tool.\n\
If you are certain the syntax is valid, then please report this error.\n\
Error: %s" filename language_name err ()
Error: %s" path language_name err ()

let exec_parser parser ~filename ~language_name raw =
let exec_parser parser ~path ~language_name raw =
let result = Angstrom.parse_string ~consume:All parser raw in
match result with
| Ok parsed -> parsed
| Error err -> default_syntax_error ~filename ~language_name ~err
| Error err -> default_syntax_error ~path ~language_name ~err

let exec_parser_lwt ?(error_message = default_error_message) parser ~filename ~language_name ic =
let exec_parser_lwt ?(error_message = default_error_message) parser ~path ~language_name ic =
let open Lwt.Infix in
Angstrom_lwt_unix.parse parser ic >|= function
| Angstrom.Buffered.{ len = 0; _ }, Ok parsed -> parsed
| Angstrom.Buffered.{ buf; off; len }, Ok _ ->
let unparsed = Bigstringaf.substring buf ~off ~len in
failwith (error_message ~filename ~language_name ~unparsed)
| _, Error err -> default_syntax_error ~filename ~language_name ~err
failwith (error_message ~path ~language_name ~unparsed)
| _, Error err -> default_syntax_error ~path ~language_name ~err
2 changes: 1 addition & 1 deletion src/parsing/js.ml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,5 +50,5 @@ let extract_to_collector (collector : Utils.Collector.t) source =
| exception Parse_error.Error (_, (_ :: _ as errors)) -> parse_error collector (First errors)
| exception Parse_error.Error (_, []) -> parse_error collector (Second "Syntax error")
| exception exn ->
print_endline (sprintf "Unexpected error in %s\nPlease report this bug." collector.filename);
print_endline (sprintf "Unexpected error in %s\nPlease report this bug." collector.path);
raise exn
8 changes: 4 additions & 4 deletions src/parsing/strings.ml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,22 +29,22 @@ let parser =
in
many (line <|> comment) <* mlws

let parse ~filename ic =
let parse ~path ic =
let open Lwt.Syntax in
let table = String.Table.create () in

let error_message ~filename ~language_name:_ ~unparsed =
let error_message ~path ~language_name:_ ~unparsed =
sprintf
"There is a syntax error in file [%s].\n\
Translations must follow this format and end in a semicolon: \"english text\" = \"translated \
text\";\n\
This line is malformed:\n\
%s"
filename
path
(String.take_while ~f:(Char.( <> ) '\n') unparsed)
in

let+ lines = Basic.exec_parser_lwt ~error_message parser ~filename ~language_name:".strings" ic in
let+ lines = Basic.exec_parser_lwt ~error_message parser ~path ~language_name:".strings" ic in
List.iter lines ~f:(function
| Translation (x, y) -> String.Table.set table ~key:x ~data:y
| Comment -> ());
Expand Down
Loading

0 comments on commit 788dbaf

Please sign in to comment.