From 7621ac28eb34a0c32a7d6cbb5e485f8f306eeea9 Mon Sep 17 00:00:00 2001 From: nohehf Date: Tue, 28 May 2024 10:47:08 +0200 Subject: [PATCH 1/9] feat: add root path and pass file path explicitely --- .../src/stack-graphs.tsg | 5 ++- .../test/root_path.py | 21 ++++++++++++ tree-sitter-stack-graphs/src/cli/index.rs | 11 ++++++- tree-sitter-stack-graphs/src/cli/test.rs | 2 ++ tree-sitter-stack-graphs/src/lib.rs | 27 +++++++++++++-- tree-sitter-stack-graphs/src/loader.rs | 33 ++++++++++++++----- tree-sitter-stack-graphs/src/test.rs | 4 +++ tree-sitter-stack-graphs/tests/it/builder.rs | 27 ++++++++++++--- tree-sitter-stack-graphs/tests/it/main.rs | 17 ++++++++-- tree-sitter-stack-graphs/tests/it/test.rs | 14 +++++++- 10 files changed, 140 insertions(+), 21 deletions(-) create mode 100644 languages/tree-sitter-stack-graphs-python/test/root_path.py diff --git a/languages/tree-sitter-stack-graphs-python/src/stack-graphs.tsg b/languages/tree-sitter-stack-graphs-python/src/stack-graphs.tsg index 6b5efd35f..887e171b3 100644 --- a/languages/tree-sitter-stack-graphs-python/src/stack-graphs.tsg +++ b/languages/tree-sitter-stack-graphs-python/src/stack-graphs.tsg @@ -13,6 +13,7 @@ ;; ^^^^^^^^^^^^^^^^ global FILE_PATH +global ROOT_PATH global ROOT_NODE global JUMP_TO_SCOPE_NODE @@ -272,7 +273,9 @@ inherit .parent_module node grandparent_module_ref_node var grandparent_module_ref = grandparent_module_ref_node - scan FILE_PATH { + ; get the file path relative to the root path + let rel_path = (replace FILE_PATH ROOT_PATH "") + scan rel_path { "([^/]+)/" { node def_dot diff --git a/languages/tree-sitter-stack-graphs-python/test/root_path.py b/languages/tree-sitter-stack-graphs-python/test/root_path.py new file mode 100644 index 000000000..23937db7c --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/root_path.py @@ -0,0 +1,21 @@ +# ------ path: foo/bar/module.py -----------# +# ------ global: ROOT_PATH=foo/bar -----------# + +foo = 42 + +# ------ path: foo/bar/baz/module.py -----------# +# ------ global: ROOT_PATH=foo/bar -----------# + +bar = "hello" + +# ------ path: foo/bar/main.py -------------# +# ------ global: ROOT_PATH=foo/bar -----------# + +from module import foo +from baz.module import bar + +print(foo) +# ^ defined: 4, 14 + +print(bar) +# ^ defined: 9, 15 diff --git a/tree-sitter-stack-graphs/src/cli/index.rs b/tree-sitter-stack-graphs/src/cli/index.rs index e8704f99d..5bf48f54f 100644 --- a/tree-sitter-stack-graphs/src/cli/index.rs +++ b/tree-sitter-stack-graphs/src/cli/index.rs @@ -428,10 +428,19 @@ impl<'a> Indexer<'a> { cancellation_flag: &dyn CancellationFlag, ) -> std::result::Result<(), BuildErrorWithSource<'b>> { let relative_source_path = source_path.strip_prefix(source_root).unwrap(); + // here the file should also have stripped the source_root from its path if let Some(lc) = lcs.primary { let globals = Variables::new(); lc.sgl - .build_stack_graph_into(graph, file, source, &globals, cancellation_flag) + .build_stack_graph_into( + graph, + file, + source, + source_path, + source_root, + &globals, + cancellation_flag, + ) .map_err(|inner| BuildErrorWithSource { inner, source_path: source_path.to_path_buf(), diff --git a/tree-sitter-stack-graphs/src/cli/test.rs b/tree-sitter-stack-graphs/src/cli/test.rs index c7f73e7ce..210d79947 100644 --- a/tree-sitter-stack-graphs/src/cli/test.rs +++ b/tree-sitter-stack-graphs/src/cli/test.rs @@ -321,6 +321,8 @@ impl TestArgs { &mut test.graph, test_fragment.file, &test_fragment.source, + &test_fragment.path, + &test_fragment.root_path, &globals, cancellation_flag.as_ref(), ) diff --git a/tree-sitter-stack-graphs/src/lib.rs b/tree-sitter-stack-graphs/src/lib.rs index 86651b6be..aee860420 100644 --- a/tree-sitter-stack-graphs/src/lib.rs +++ b/tree-sitter-stack-graphs/src/lib.rs @@ -438,6 +438,7 @@ static PRECEDENCE_ATTR: &'static str = "precedence"; static ROOT_NODE_VAR: &'static str = "ROOT_NODE"; static JUMP_TO_SCOPE_NODE_VAR: &'static str = "JUMP_TO_SCOPE_NODE"; static FILE_PATH_VAR: &'static str = "FILE_PATH"; +static ROOT_PATH_VAR: &'static str = "ROOT_PATH"; /// Holds information about how to construct stack graphs for a particular language. pub struct StackGraphLanguage { @@ -557,10 +558,12 @@ impl StackGraphLanguage { stack_graph: &'a mut StackGraph, file: Handle, source: &'a str, + source_path: &Path, + source_root: &Path, globals: &'a Variables<'a>, cancellation_flag: &'a dyn CancellationFlag, ) -> Result<(), BuildError> { - self.builder_into_stack_graph(stack_graph, file, source) + self.builder_into_stack_graph(stack_graph, file, source, source_path, source_root) .build(globals, cancellation_flag) } @@ -573,8 +576,10 @@ impl StackGraphLanguage { stack_graph: &'a mut StackGraph, file: Handle, source: &'a str, + source_path: &'a Path, + source_root: &'a Path, ) -> Builder<'a> { - Builder::new(self, stack_graph, file, source) + Builder::new(self, stack_graph, file, source, source_path, source_root) } } @@ -583,6 +588,8 @@ pub struct Builder<'a> { stack_graph: &'a mut StackGraph, file: Handle, source: &'a str, + source_path: &'a Path, + source_root: &'a Path, graph: Graph<'a>, remapped_nodes: HashMap, injected_node_count: usize, @@ -595,6 +602,8 @@ impl<'a> Builder<'a> { stack_graph: &'a mut StackGraph, file: Handle, source: &'a str, + source_path: &'a Path, + source_root: &'a Path, ) -> Self { let span_calculator = SpanCalculator::new(source); Builder { @@ -602,6 +611,8 @@ impl<'a> Builder<'a> { stack_graph, file, source, + source_path, + source_root, graph: Graph::new(), remapped_nodes: HashMap::new(), injected_node_count: 0, @@ -635,23 +646,33 @@ impl<'a> Builder<'a> { let tree = parse_errors.into_tree(); let mut globals = Variables::nested(globals); + if globals.get(&ROOT_NODE_VAR.into()).is_none() { let root_node = self.inject_node(NodeID::root()); globals .add(ROOT_NODE_VAR.into(), root_node.into()) .expect("Failed to set ROOT_NODE"); } + let jump_to_scope_node = self.inject_node(NodeID::jump_to()); globals .add(JUMP_TO_SCOPE_NODE_VAR.into(), jump_to_scope_node.into()) .expect("Failed to set JUMP_TO_SCOPE_NODE"); + if globals.get(&FILE_PATH_VAR.into()).is_none() { - let file_name = self.stack_graph[self.file].to_string(); + let file_name = self.source_path.to_str().unwrap().to_string(); globals .add(FILE_PATH_VAR.into(), file_name.into()) .expect("Failed to set FILE_PATH"); } + if globals.get(&ROOT_PATH_VAR.into()).is_none() { + let root_path = self.source_root.to_str().unwrap().to_string(); + globals + .add(ROOT_PATH_VAR.into(), root_path.into()) + .expect("Failed to set ROOT_PATH"); + } + let mut config = ExecutionConfig::new(&self.sgl.functions, &globals) .lazy(true) .debug_attributes( diff --git a/tree-sitter-stack-graphs/src/loader.rs b/tree-sitter-stack-graphs/src/loader.rs index 9ffc1d674..ced44c2ed 100644 --- a/tree-sitter-stack-graphs/src/loader.rs +++ b/tree-sitter-stack-graphs/src/loader.rs @@ -79,10 +79,14 @@ impl LanguageConfiguration { Loader::load_globals_from_config_str(builtins_config, &mut builtins_globals)?; } let file = builtins.add_file("").unwrap(); + let builtins_path_var = Path::new(""); + let builtins_root = Path::new(""); sgl.build_stack_graph_into( &mut builtins, file, builtins_source, + builtins_path_var, + builtins_root, &builtins_globals, cancellation_flag, ) @@ -325,17 +329,28 @@ impl Loader { graph: &mut StackGraph, cancellation_flag: &dyn CancellationFlag, ) -> Result<(), LoadError<'a>> { - let file = graph.add_file(&path.to_string_lossy()).unwrap(); + let file_name = path.to_string_lossy(); + let file: stack_graphs::arena::Handle = + graph.add_file(&file_name).unwrap(); + let builtins_root = Path::new(""); let mut globals = Variables::new(); Self::load_globals_from_config_str(&config, &mut globals)?; - sgl.build_stack_graph_into(graph, file, &source, &globals, cancellation_flag) - .map_err(|err| LoadError::Builtins { - inner: err, - source_path: path.to_path_buf(), - source, - tsg_path: sgl.tsg_path.to_path_buf(), - tsg: sgl.tsg_source.clone(), - })?; + sgl.build_stack_graph_into( + graph, + file, + &source, + path, + builtins_root, + &globals, + cancellation_flag, + ) + .map_err(|err| LoadError::Builtins { + inner: err, + source_path: path.to_path_buf(), + source, + tsg_path: sgl.tsg_path.to_path_buf(), + tsg: sgl.tsg_source.clone(), + })?; return Ok(()); } diff --git a/tree-sitter-stack-graphs/src/test.rs b/tree-sitter-stack-graphs/src/test.rs index 236e254c5..554783b5f 100644 --- a/tree-sitter-stack-graphs/src/test.rs +++ b/tree-sitter-stack-graphs/src/test.rs @@ -157,6 +157,7 @@ pub struct Test { pub struct TestFragment { pub file: Handle, pub path: PathBuf, + pub root_path: PathBuf, pub source: String, pub assertions: Vec, pub globals: HashMap, @@ -180,6 +181,7 @@ impl Test { let mut prev_source = String::new(); let mut line_files = Vec::new(); let mut line_count = 0; + let default_root_path = PathBuf::from(""); for (current_line_number, current_line) in PositionedSubstring::lines_iter(source).enumerate() { @@ -202,6 +204,7 @@ impl Test { fragments.push(TestFragment { file, path: current_path, + root_path: default_root_path.clone(), source: current_source, assertions: Vec::new(), globals: current_globals, @@ -254,6 +257,7 @@ impl Test { fragments.push(TestFragment { file, path: current_path, + root_path: default_root_path.clone(), source: current_source, assertions: Vec::new(), globals: current_globals, diff --git a/tree-sitter-stack-graphs/tests/it/builder.rs b/tree-sitter-stack-graphs/tests/it/builder.rs index 661e8b973..31dc46994 100644 --- a/tree-sitter-stack-graphs/tests/it/builder.rs +++ b/tree-sitter-stack-graphs/tests/it/builder.rs @@ -5,6 +5,8 @@ // Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. // ------------------------------------------------------------------------------------------------ +use std::path::Path; + use stack_graphs::graph::StackGraph; use tree_sitter_graph::Variables; use tree_sitter_stack_graphs::NoCancellation; @@ -22,15 +24,27 @@ fn can_support_preexisting_nodes() { "#; let python = "pass"; + let file_name = "test.py"; + let source_path = Path::new(file_name); + let source_root = Path::new(""); + let mut graph = StackGraph::new(); - let file = graph.get_or_create_file("test.py"); + let file = graph.get_or_create_file(file_name); let node_id = graph.new_node_id(file); let _preexisting_node = graph.add_scope_node(node_id, true).unwrap(); let globals = Variables::new(); let language = StackGraphLanguage::from_str(tree_sitter_python::language(), tsg).unwrap(); language - .build_stack_graph_into(&mut graph, file, python, &globals, &NoCancellation) + .build_stack_graph_into( + &mut graph, + file, + python, + source_path, + source_root, + &globals, + &NoCancellation, + ) .expect("Failed to build graph"); } @@ -45,13 +59,18 @@ fn can_support_injected_nodes() { "#; let python = "pass"; + let file_name = "test.py"; + let source_path = Path::new(file_name); + let source_root = Path::new(""); + let mut graph = StackGraph::new(); - let file = graph.get_or_create_file("test.py"); + let file = graph.get_or_create_file(file_name); let node_id = graph.new_node_id(file); let _preexisting_node = graph.add_scope_node(node_id, true).unwrap(); let language = StackGraphLanguage::from_str(tree_sitter_python::language(), tsg).unwrap(); - let mut builder = language.builder_into_stack_graph(&mut graph, file, python); + let mut builder = + language.builder_into_stack_graph(&mut graph, file, python, source_path, source_root); let mut globals = Variables::new(); globals diff --git a/tree-sitter-stack-graphs/tests/it/main.rs b/tree-sitter-stack-graphs/tests/it/main.rs index c1e37a40e..9d430bb73 100644 --- a/tree-sitter-stack-graphs/tests/it/main.rs +++ b/tree-sitter-stack-graphs/tests/it/main.rs @@ -5,6 +5,8 @@ // Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. // ------------------------------------------------------------------------------------------------ +use std::path::Path; + use stack_graphs::arena::Handle; use stack_graphs::graph::File; use stack_graphs::graph::StackGraph; @@ -23,11 +25,22 @@ pub(self) fn build_stack_graph( python_source: &str, tsg_source: &str, ) -> Result<(StackGraph, Handle), BuildError> { + let file_name = "test.py"; + let source_path = Path::new(file_name); + let source_root = Path::new(""); let language = StackGraphLanguage::from_str(tree_sitter_python::language(), tsg_source).unwrap(); let mut graph = StackGraph::new(); - let file = graph.get_or_create_file("test.py"); + let file = graph.get_or_create_file(file_name); let globals = Variables::new(); - language.build_stack_graph_into(&mut graph, file, python_source, &globals, &NoCancellation)?; + language.build_stack_graph_into( + &mut graph, + file, + python_source, + source_path, + source_root, + &globals, + &NoCancellation, + )?; Ok((graph, file)) } diff --git a/tree-sitter-stack-graphs/tests/it/test.rs b/tree-sitter-stack-graphs/tests/it/test.rs index 98ec3ab96..89ae7b279 100644 --- a/tree-sitter-stack-graphs/tests/it/test.rs +++ b/tree-sitter-stack-graphs/tests/it/test.rs @@ -68,12 +68,22 @@ fn build_stack_graph_into( graph: &mut StackGraph, file: Handle, python_source: &str, + source_path: &Path, + source_root: &Path, tsg_source: &str, globals: &Variables, ) -> Result<(), BuildError> { let language = StackGraphLanguage::from_str(tree_sitter_python::language(), tsg_source).unwrap(); - language.build_stack_graph_into(graph, file, python_source, globals, &NoCancellation)?; + language.build_stack_graph_into( + graph, + file, + python_source, + source_path, + source_root, + globals, + &NoCancellation, + )?; Ok(()) } @@ -102,6 +112,8 @@ fn check_test( &mut test.graph, fragments.file, &fragments.source, + &fragments.path, + &fragments.root_path, tsg_source, &globals, ) From b3260536a1002743a9a393a64ce8fd33ca859b70 Mon Sep 17 00:00:00 2001 From: nohehf Date: Thu, 30 May 2024 12:17:15 +0200 Subject: [PATCH 2/9] feat: pass paths in globals --- .../src/stack-graphs.tsg | 2 +- tree-sitter-stack-graphs/src/cli/index.rs | 23 +++++----- tree-sitter-stack-graphs/src/cli/test.rs | 12 ++++- tree-sitter-stack-graphs/src/lib.rs | 31 +++---------- tree-sitter-stack-graphs/src/loader.rs | 44 ++++++++++--------- tree-sitter-stack-graphs/src/test.rs | 4 -- tree-sitter-stack-graphs/tests/it/builder.rs | 13 +----- tree-sitter-stack-graphs/tests/it/main.rs | 23 +++++----- tree-sitter-stack-graphs/tests/it/test.rs | 14 +----- 9 files changed, 66 insertions(+), 100 deletions(-) diff --git a/languages/tree-sitter-stack-graphs-python/src/stack-graphs.tsg b/languages/tree-sitter-stack-graphs-python/src/stack-graphs.tsg index 887e171b3..5fb407ae8 100644 --- a/languages/tree-sitter-stack-graphs-python/src/stack-graphs.tsg +++ b/languages/tree-sitter-stack-graphs-python/src/stack-graphs.tsg @@ -13,7 +13,7 @@ ;; ^^^^^^^^^^^^^^^^ global FILE_PATH -global ROOT_PATH +global ROOT_PATH = "" global ROOT_NODE global JUMP_TO_SCOPE_NODE diff --git a/tree-sitter-stack-graphs/src/cli/index.rs b/tree-sitter-stack-graphs/src/cli/index.rs index 5bf48f54f..6425d8ca7 100644 --- a/tree-sitter-stack-graphs/src/cli/index.rs +++ b/tree-sitter-stack-graphs/src/cli/index.rs @@ -42,6 +42,7 @@ use crate::BuildError; use crate::CancelAfterDuration; use crate::CancellationFlag; use crate::NoCancellation; +use crate::{FILE_PATH_VAR, ROOT_PATH_VAR}; #[derive(Args)] pub struct IndexArgs { @@ -428,19 +429,19 @@ impl<'a> Indexer<'a> { cancellation_flag: &dyn CancellationFlag, ) -> std::result::Result<(), BuildErrorWithSource<'b>> { let relative_source_path = source_path.strip_prefix(source_root).unwrap(); - // here the file should also have stripped the source_root from its path if let Some(lc) = lcs.primary { - let globals = Variables::new(); + let mut globals = Variables::new(); + + globals + .add(FILE_PATH_VAR.into(), source_path.to_str().unwrap().into()) + .expect("failed to add file path variable"); + + globals + .add(ROOT_PATH_VAR.into(), source_root.to_str().unwrap().into()) + .expect("failed to add root path variable"); + lc.sgl - .build_stack_graph_into( - graph, - file, - source, - source_path, - source_root, - &globals, - cancellation_flag, - ) + .build_stack_graph_into(graph, file, source, &globals, cancellation_flag) .map_err(|inner| BuildErrorWithSource { inner, source_path: source_path.to_path_buf(), diff --git a/tree-sitter-stack-graphs/src/cli/test.rs b/tree-sitter-stack-graphs/src/cli/test.rs index 210d79947..6ae04114c 100644 --- a/tree-sitter-stack-graphs/src/cli/test.rs +++ b/tree-sitter-stack-graphs/src/cli/test.rs @@ -43,6 +43,7 @@ use crate::test::Test; use crate::test::TestResult; use crate::CancelAfterDuration; use crate::CancellationFlag; +use crate::FILE_PATH_VAR; #[derive(Args)] #[clap(after_help = r#"PATH SPECIFICATIONS: @@ -316,13 +317,20 @@ impl TestArgs { &mut Some(test_fragment.source.as_ref()), )? { globals.clear(); + + // Add FILE_PATH to variables + globals + .add( + FILE_PATH_VAR.into(), + test_fragment.path.to_str().unwrap().into(), + ) + .expect("failed to add file path variable"); + test_fragment.add_globals_to(&mut globals); lc.sgl.build_stack_graph_into( &mut test.graph, test_fragment.file, &test_fragment.source, - &test_fragment.path, - &test_fragment.root_path, &globals, cancellation_flag.as_ref(), ) diff --git a/tree-sitter-stack-graphs/src/lib.rs b/tree-sitter-stack-graphs/src/lib.rs index aee860420..8a02b0d58 100644 --- a/tree-sitter-stack-graphs/src/lib.rs +++ b/tree-sitter-stack-graphs/src/lib.rs @@ -558,12 +558,10 @@ impl StackGraphLanguage { stack_graph: &'a mut StackGraph, file: Handle, source: &'a str, - source_path: &Path, - source_root: &Path, globals: &'a Variables<'a>, cancellation_flag: &'a dyn CancellationFlag, ) -> Result<(), BuildError> { - self.builder_into_stack_graph(stack_graph, file, source, source_path, source_root) + self.builder_into_stack_graph(stack_graph, file, source) .build(globals, cancellation_flag) } @@ -576,10 +574,8 @@ impl StackGraphLanguage { stack_graph: &'a mut StackGraph, file: Handle, source: &'a str, - source_path: &'a Path, - source_root: &'a Path, ) -> Builder<'a> { - Builder::new(self, stack_graph, file, source, source_path, source_root) + Builder::new(self, stack_graph, file, source) } } @@ -588,8 +584,6 @@ pub struct Builder<'a> { stack_graph: &'a mut StackGraph, file: Handle, source: &'a str, - source_path: &'a Path, - source_root: &'a Path, graph: Graph<'a>, remapped_nodes: HashMap, injected_node_count: usize, @@ -602,8 +596,6 @@ impl<'a> Builder<'a> { stack_graph: &'a mut StackGraph, file: Handle, source: &'a str, - source_path: &'a Path, - source_root: &'a Path, ) -> Self { let span_calculator = SpanCalculator::new(source); Builder { @@ -611,8 +603,6 @@ impl<'a> Builder<'a> { stack_graph, file, source, - source_path, - source_root, graph: Graph::new(), remapped_nodes: HashMap::new(), injected_node_count: 0, @@ -659,19 +649,10 @@ impl<'a> Builder<'a> { .add(JUMP_TO_SCOPE_NODE_VAR.into(), jump_to_scope_node.into()) .expect("Failed to set JUMP_TO_SCOPE_NODE"); - if globals.get(&FILE_PATH_VAR.into()).is_none() { - let file_name = self.source_path.to_str().unwrap().to_string(); - globals - .add(FILE_PATH_VAR.into(), file_name.into()) - .expect("Failed to set FILE_PATH"); - } - - if globals.get(&ROOT_PATH_VAR.into()).is_none() { - let root_path = self.source_root.to_str().unwrap().to_string(); - globals - .add(ROOT_PATH_VAR.into(), root_path.into()) - .expect("Failed to set ROOT_PATH"); - } + // FILE_PATH is mandatory + globals + .get(&FILE_PATH_VAR.into()) + .expect("FILE_PATH not set"); let mut config = ExecutionConfig::new(&self.sgl.functions, &globals) .lazy(true) diff --git a/tree-sitter-stack-graphs/src/loader.rs b/tree-sitter-stack-graphs/src/loader.rs index ced44c2ed..b0d6d1875 100644 --- a/tree-sitter-stack-graphs/src/loader.rs +++ b/tree-sitter-stack-graphs/src/loader.rs @@ -29,6 +29,7 @@ use tree_sitter_loader::Loader as TsLoader; use crate::CancellationFlag; use crate::FileAnalyzer; use crate::StackGraphLanguage; +use crate::FILE_PATH_VAR; pub static DEFAULT_TSG_PATHS: Lazy> = Lazy::new(|| vec![LoadPath::Grammar("queries/stack-graphs".into())]); @@ -75,18 +76,23 @@ impl LanguageConfiguration { let mut builtins = StackGraph::new(); if let Some((builtins_path, builtins_source)) = builtins_source { let mut builtins_globals = Variables::new(); + + let builtins_path_var = Path::new(""); + builtins_globals + .add( + FILE_PATH_VAR.into(), + builtins_path_var.to_str().unwrap().into(), + ) + .expect("failed to add file path variable"); + if let Some(builtins_config) = builtins_config { Loader::load_globals_from_config_str(builtins_config, &mut builtins_globals)?; } let file = builtins.add_file("").unwrap(); - let builtins_path_var = Path::new(""); - let builtins_root = Path::new(""); sgl.build_stack_graph_into( &mut builtins, file, builtins_source, - builtins_path_var, - builtins_root, &builtins_globals, cancellation_flag, ) @@ -332,25 +338,21 @@ impl Loader { let file_name = path.to_string_lossy(); let file: stack_graphs::arena::Handle = graph.add_file(&file_name).unwrap(); - let builtins_root = Path::new(""); let mut globals = Variables::new(); + + globals + .add(FILE_PATH_VAR.into(), path.to_str().unwrap().into()) + .expect("failed to add file path variable"); + Self::load_globals_from_config_str(&config, &mut globals)?; - sgl.build_stack_graph_into( - graph, - file, - &source, - path, - builtins_root, - &globals, - cancellation_flag, - ) - .map_err(|err| LoadError::Builtins { - inner: err, - source_path: path.to_path_buf(), - source, - tsg_path: sgl.tsg_path.to_path_buf(), - tsg: sgl.tsg_source.clone(), - })?; + sgl.build_stack_graph_into(graph, file, &source, &globals, cancellation_flag) + .map_err(|err| LoadError::Builtins { + inner: err, + source_path: path.to_path_buf(), + source, + tsg_path: sgl.tsg_path.to_path_buf(), + tsg: sgl.tsg_source.clone(), + })?; return Ok(()); } diff --git a/tree-sitter-stack-graphs/src/test.rs b/tree-sitter-stack-graphs/src/test.rs index 554783b5f..236e254c5 100644 --- a/tree-sitter-stack-graphs/src/test.rs +++ b/tree-sitter-stack-graphs/src/test.rs @@ -157,7 +157,6 @@ pub struct Test { pub struct TestFragment { pub file: Handle, pub path: PathBuf, - pub root_path: PathBuf, pub source: String, pub assertions: Vec, pub globals: HashMap, @@ -181,7 +180,6 @@ impl Test { let mut prev_source = String::new(); let mut line_files = Vec::new(); let mut line_count = 0; - let default_root_path = PathBuf::from(""); for (current_line_number, current_line) in PositionedSubstring::lines_iter(source).enumerate() { @@ -204,7 +202,6 @@ impl Test { fragments.push(TestFragment { file, path: current_path, - root_path: default_root_path.clone(), source: current_source, assertions: Vec::new(), globals: current_globals, @@ -257,7 +254,6 @@ impl Test { fragments.push(TestFragment { file, path: current_path, - root_path: default_root_path.clone(), source: current_source, assertions: Vec::new(), globals: current_globals, diff --git a/tree-sitter-stack-graphs/tests/it/builder.rs b/tree-sitter-stack-graphs/tests/it/builder.rs index 31dc46994..a87b80559 100644 --- a/tree-sitter-stack-graphs/tests/it/builder.rs +++ b/tree-sitter-stack-graphs/tests/it/builder.rs @@ -36,15 +36,7 @@ fn can_support_preexisting_nodes() { let globals = Variables::new(); let language = StackGraphLanguage::from_str(tree_sitter_python::language(), tsg).unwrap(); language - .build_stack_graph_into( - &mut graph, - file, - python, - source_path, - source_root, - &globals, - &NoCancellation, - ) + .build_stack_graph_into(&mut graph, file, python, &globals, &NoCancellation) .expect("Failed to build graph"); } @@ -69,8 +61,7 @@ fn can_support_injected_nodes() { let _preexisting_node = graph.add_scope_node(node_id, true).unwrap(); let language = StackGraphLanguage::from_str(tree_sitter_python::language(), tsg).unwrap(); - let mut builder = - language.builder_into_stack_graph(&mut graph, file, python, source_path, source_root); + let mut builder = language.builder_into_stack_graph(&mut graph, file, python); let mut globals = Variables::new(); globals diff --git a/tree-sitter-stack-graphs/tests/it/main.rs b/tree-sitter-stack-graphs/tests/it/main.rs index 9d430bb73..687564e1e 100644 --- a/tree-sitter-stack-graphs/tests/it/main.rs +++ b/tree-sitter-stack-graphs/tests/it/main.rs @@ -15,6 +15,8 @@ use tree_sitter_stack_graphs::BuildError; use tree_sitter_stack_graphs::NoCancellation; use tree_sitter_stack_graphs::StackGraphLanguage; +static FILE_PATH_VAR: &'static str = "FILE_PATH"; + mod builder; mod edges; mod loader; @@ -26,21 +28,18 @@ pub(self) fn build_stack_graph( tsg_source: &str, ) -> Result<(StackGraph, Handle), BuildError> { let file_name = "test.py"; - let source_path = Path::new(file_name); - let source_root = Path::new(""); let language = StackGraphLanguage::from_str(tree_sitter_python::language(), tsg_source).unwrap(); let mut graph = StackGraph::new(); let file = graph.get_or_create_file(file_name); - let globals = Variables::new(); - language.build_stack_graph_into( - &mut graph, - file, - python_source, - source_path, - source_root, - &globals, - &NoCancellation, - )?; + let mut globals = Variables::new(); + let source_path = Path::new(file_name); + + // The FILE_PATH_VAR is not accessible here + globals + .add(FILE_PATH_VAR.into(), source_path.to_str().unwrap().into()) + .expect("failed to add file path variable"); + + language.build_stack_graph_into(&mut graph, file, python_source, &globals, &NoCancellation)?; Ok((graph, file)) } diff --git a/tree-sitter-stack-graphs/tests/it/test.rs b/tree-sitter-stack-graphs/tests/it/test.rs index 89ae7b279..98ec3ab96 100644 --- a/tree-sitter-stack-graphs/tests/it/test.rs +++ b/tree-sitter-stack-graphs/tests/it/test.rs @@ -68,22 +68,12 @@ fn build_stack_graph_into( graph: &mut StackGraph, file: Handle, python_source: &str, - source_path: &Path, - source_root: &Path, tsg_source: &str, globals: &Variables, ) -> Result<(), BuildError> { let language = StackGraphLanguage::from_str(tree_sitter_python::language(), tsg_source).unwrap(); - language.build_stack_graph_into( - graph, - file, - python_source, - source_path, - source_root, - globals, - &NoCancellation, - )?; + language.build_stack_graph_into(graph, file, python_source, globals, &NoCancellation)?; Ok(()) } @@ -112,8 +102,6 @@ fn check_test( &mut test.graph, fragments.file, &fragments.source, - &fragments.path, - &fragments.root_path, tsg_source, &globals, ) From 8a98dce32fee057bc4d754ebc85055fecbbda0a1 Mon Sep 17 00:00:00 2001 From: nohehf Date: Thu, 30 May 2024 12:33:37 +0200 Subject: [PATCH 3/9] fix: add missing paths --- tree-sitter-stack-graphs/tests/it/builder.rs | 15 ++++++++++----- tree-sitter-stack-graphs/tests/it/test.rs | 10 ++++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/tree-sitter-stack-graphs/tests/it/builder.rs b/tree-sitter-stack-graphs/tests/it/builder.rs index a87b80559..b0dad5449 100644 --- a/tree-sitter-stack-graphs/tests/it/builder.rs +++ b/tree-sitter-stack-graphs/tests/it/builder.rs @@ -14,6 +14,7 @@ use tree_sitter_stack_graphs::StackGraphLanguage; use crate::edges::check_stack_graph_edges; use crate::nodes::check_stack_graph_nodes; +use crate::FILE_PATH_VAR; #[test] fn can_support_preexisting_nodes() { @@ -25,15 +26,17 @@ fn can_support_preexisting_nodes() { let python = "pass"; let file_name = "test.py"; - let source_path = Path::new(file_name); - let source_root = Path::new(""); let mut graph = StackGraph::new(); let file = graph.get_or_create_file(file_name); let node_id = graph.new_node_id(file); let _preexisting_node = graph.add_scope_node(node_id, true).unwrap(); - let globals = Variables::new(); + let mut globals = Variables::new(); + globals + .add(FILE_PATH_VAR.into(), file_name.into()) + .expect("failed to add file path variable"); + let language = StackGraphLanguage::from_str(tree_sitter_python::language(), tsg).unwrap(); language .build_stack_graph_into(&mut graph, file, python, &globals, &NoCancellation) @@ -52,8 +55,6 @@ fn can_support_injected_nodes() { let python = "pass"; let file_name = "test.py"; - let source_path = Path::new(file_name); - let source_root = Path::new(""); let mut graph = StackGraph::new(); let file = graph.get_or_create_file(file_name); @@ -64,6 +65,10 @@ fn can_support_injected_nodes() { let mut builder = language.builder_into_stack_graph(&mut graph, file, python); let mut globals = Variables::new(); + globals + .add(FILE_PATH_VAR.into(), file_name.into()) + .expect("failed to add file path variable"); + globals .add("EXT_NODE".into(), builder.inject_node(node_id).into()) .expect("Failed to add EXT_NODE variable"); diff --git a/tree-sitter-stack-graphs/tests/it/test.rs b/tree-sitter-stack-graphs/tests/it/test.rs index 98ec3ab96..9624439c5 100644 --- a/tree-sitter-stack-graphs/tests/it/test.rs +++ b/tree-sitter-stack-graphs/tests/it/test.rs @@ -5,6 +5,7 @@ // Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. // ------------------------------------------------------------------------------------------------ +use crate::FILE_PATH_VAR; use once_cell::sync::Lazy; use pretty_assertions::assert_eq; use stack_graphs::arena::Handle; @@ -94,9 +95,18 @@ fn check_test( expected_successes + expected_failures, assertion_count, ); + let mut globals = Variables::new(); for fragments in &test.fragments { globals.clear(); + + globals + .add( + FILE_PATH_VAR.into(), + fragments.path.to_str().unwrap().into(), + ) + .expect("failed to add file path variable"); + fragments.add_globals_to(&mut globals); build_stack_graph_into( &mut test.graph, From ad4d517bf52b0615d7798f106e77bbd592ac694f Mon Sep 17 00:00:00 2001 From: nohehf Date: Sat, 1 Jun 2024 14:47:32 +0200 Subject: [PATCH 4/9] chore: clenup after review --- tree-sitter-stack-graphs/src/lib.rs | 19 ++++++++++--------- tree-sitter-stack-graphs/src/loader.rs | 12 +++++------- tree-sitter-stack-graphs/tests/it/builder.rs | 5 +---- tree-sitter-stack-graphs/tests/it/main.rs | 4 +--- 4 files changed, 17 insertions(+), 23 deletions(-) diff --git a/tree-sitter-stack-graphs/src/lib.rs b/tree-sitter-stack-graphs/src/lib.rs index 8a02b0d58..1a3e344b1 100644 --- a/tree-sitter-stack-graphs/src/lib.rs +++ b/tree-sitter-stack-graphs/src/lib.rs @@ -435,10 +435,16 @@ static SCOPE_ATTRS: Lazy> = static PRECEDENCE_ATTR: &'static str = "precedence"; // Global variables -static ROOT_NODE_VAR: &'static str = "ROOT_NODE"; -static JUMP_TO_SCOPE_NODE_VAR: &'static str = "JUMP_TO_SCOPE_NODE"; -static FILE_PATH_VAR: &'static str = "FILE_PATH"; -static ROOT_PATH_VAR: &'static str = "ROOT_PATH"; +/// Name of the variable used to pass the root node. +pub const ROOT_NODE_VAR: &'static str = "ROOT_NODE"; +/// Name of the variable used to pass the jump-to-scope node. +pub const JUMP_TO_SCOPE_NODE_VAR: &'static str = "JUMP_TO_SCOPE_NODE"; +/// Name of the variable used to pass the file path. +/// If a root path is given, it should be a descendant the root path. +pub const FILE_PATH_VAR: &'static str = "FILE_PATH"; +/// Name of the variable used to pass the root path. +/// If given, should be an ancestor of the file path. +pub const ROOT_PATH_VAR: &'static str = "ROOT_PATH"; /// Holds information about how to construct stack graphs for a particular language. pub struct StackGraphLanguage { @@ -649,11 +655,6 @@ impl<'a> Builder<'a> { .add(JUMP_TO_SCOPE_NODE_VAR.into(), jump_to_scope_node.into()) .expect("Failed to set JUMP_TO_SCOPE_NODE"); - // FILE_PATH is mandatory - globals - .get(&FILE_PATH_VAR.into()) - .expect("FILE_PATH not set"); - let mut config = ExecutionConfig::new(&self.sgl.functions, &globals) .lazy(true) .debug_attributes( diff --git a/tree-sitter-stack-graphs/src/loader.rs b/tree-sitter-stack-graphs/src/loader.rs index b0d6d1875..f0ab9891c 100644 --- a/tree-sitter-stack-graphs/src/loader.rs +++ b/tree-sitter-stack-graphs/src/loader.rs @@ -31,6 +31,8 @@ use crate::FileAnalyzer; use crate::StackGraphLanguage; use crate::FILE_PATH_VAR; +const BUILTINS_FILENAME: &str = ""; + pub static DEFAULT_TSG_PATHS: Lazy> = Lazy::new(|| vec![LoadPath::Grammar("queries/stack-graphs".into())]); pub static DEFAULT_BUILTINS_PATHS: Lazy> = @@ -77,18 +79,14 @@ impl LanguageConfiguration { if let Some((builtins_path, builtins_source)) = builtins_source { let mut builtins_globals = Variables::new(); - let builtins_path_var = Path::new(""); builtins_globals - .add( - FILE_PATH_VAR.into(), - builtins_path_var.to_str().unwrap().into(), - ) + .add(FILE_PATH_VAR.into(), BUILTINS_FILENAME.into()) .expect("failed to add file path variable"); if let Some(builtins_config) = builtins_config { Loader::load_globals_from_config_str(builtins_config, &mut builtins_globals)?; } - let file = builtins.add_file("").unwrap(); + let file = builtins.add_file(BUILTINS_FILENAME).unwrap(); sgl.build_stack_graph_into( &mut builtins, file, @@ -341,7 +339,7 @@ impl Loader { let mut globals = Variables::new(); globals - .add(FILE_PATH_VAR.into(), path.to_str().unwrap().into()) + .add(FILE_PATH_VAR.into(), BUILTINS_FILENAME.into()) .expect("failed to add file path variable"); Self::load_globals_from_config_str(&config, &mut globals)?; diff --git a/tree-sitter-stack-graphs/tests/it/builder.rs b/tree-sitter-stack-graphs/tests/it/builder.rs index b0dad5449..acc012eaa 100644 --- a/tree-sitter-stack-graphs/tests/it/builder.rs +++ b/tree-sitter-stack-graphs/tests/it/builder.rs @@ -4,17 +4,14 @@ // Licensed under either of Apache License, Version 2.0, or MIT license, at your option. // Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. // ------------------------------------------------------------------------------------------------ - -use std::path::Path; - use stack_graphs::graph::StackGraph; use tree_sitter_graph::Variables; use tree_sitter_stack_graphs::NoCancellation; use tree_sitter_stack_graphs::StackGraphLanguage; +use tree_sitter_stack_graphs::FILE_PATH_VAR; use crate::edges::check_stack_graph_edges; use crate::nodes::check_stack_graph_nodes; -use crate::FILE_PATH_VAR; #[test] fn can_support_preexisting_nodes() { diff --git a/tree-sitter-stack-graphs/tests/it/main.rs b/tree-sitter-stack-graphs/tests/it/main.rs index 687564e1e..33ee19241 100644 --- a/tree-sitter-stack-graphs/tests/it/main.rs +++ b/tree-sitter-stack-graphs/tests/it/main.rs @@ -14,8 +14,7 @@ use tree_sitter_graph::Variables; use tree_sitter_stack_graphs::BuildError; use tree_sitter_stack_graphs::NoCancellation; use tree_sitter_stack_graphs::StackGraphLanguage; - -static FILE_PATH_VAR: &'static str = "FILE_PATH"; +use tree_sitter_stack_graphs::FILE_PATH_VAR; mod builder; mod edges; @@ -35,7 +34,6 @@ pub(self) fn build_stack_graph( let mut globals = Variables::new(); let source_path = Path::new(file_name); - // The FILE_PATH_VAR is not accessible here globals .add(FILE_PATH_VAR.into(), source_path.to_str().unwrap().into()) .expect("failed to add file path variable"); From 5ba472ab81783908360aaf3d1b8b15d2e2b85534 Mon Sep 17 00:00:00 2001 From: nohehf Date: Sat, 1 Jun 2024 23:31:00 +0200 Subject: [PATCH 5/9] style: style --- tree-sitter-stack-graphs/src/loader.rs | 3 +-- tree-sitter-stack-graphs/tests/it/builder.rs | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tree-sitter-stack-graphs/src/loader.rs b/tree-sitter-stack-graphs/src/loader.rs index f0ab9891c..e0e1d236d 100644 --- a/tree-sitter-stack-graphs/src/loader.rs +++ b/tree-sitter-stack-graphs/src/loader.rs @@ -334,8 +334,7 @@ impl Loader { cancellation_flag: &dyn CancellationFlag, ) -> Result<(), LoadError<'a>> { let file_name = path.to_string_lossy(); - let file: stack_graphs::arena::Handle = - graph.add_file(&file_name).unwrap(); + let file = graph.add_file(&file_name).unwrap(); let mut globals = Variables::new(); globals diff --git a/tree-sitter-stack-graphs/tests/it/builder.rs b/tree-sitter-stack-graphs/tests/it/builder.rs index acc012eaa..14476111e 100644 --- a/tree-sitter-stack-graphs/tests/it/builder.rs +++ b/tree-sitter-stack-graphs/tests/it/builder.rs @@ -4,6 +4,7 @@ // Licensed under either of Apache License, Version 2.0, or MIT license, at your option. // Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. // ------------------------------------------------------------------------------------------------ + use stack_graphs::graph::StackGraph; use tree_sitter_graph::Variables; use tree_sitter_stack_graphs::NoCancellation; From ba1d3cd125520583a4e56f5e11c92e34f1f855cf Mon Sep 17 00:00:00 2001 From: nohehf Date: Sun, 2 Jun 2024 00:09:42 +0200 Subject: [PATCH 6/9] fix: conditionnaly load FILE_PATH --- tree-sitter-stack-graphs/src/loader.rs | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/tree-sitter-stack-graphs/src/loader.rs b/tree-sitter-stack-graphs/src/loader.rs index e0e1d236d..d382f72c9 100644 --- a/tree-sitter-stack-graphs/src/loader.rs +++ b/tree-sitter-stack-graphs/src/loader.rs @@ -79,13 +79,16 @@ impl LanguageConfiguration { if let Some((builtins_path, builtins_source)) = builtins_source { let mut builtins_globals = Variables::new(); - builtins_globals - .add(FILE_PATH_VAR.into(), BUILTINS_FILENAME.into()) - .expect("failed to add file path variable"); - if let Some(builtins_config) = builtins_config { Loader::load_globals_from_config_str(builtins_config, &mut builtins_globals)?; } + + if builtins_globals.get(&FILE_PATH_VAR.into()).is_some() { + builtins_globals + .add(FILE_PATH_VAR.into(), BUILTINS_FILENAME.into()) + .expect("failed to add file path variable"); + } + let file = builtins.add_file(BUILTINS_FILENAME).unwrap(); sgl.build_stack_graph_into( &mut builtins, @@ -337,11 +340,14 @@ impl Loader { let file = graph.add_file(&file_name).unwrap(); let mut globals = Variables::new(); - globals - .add(FILE_PATH_VAR.into(), BUILTINS_FILENAME.into()) - .expect("failed to add file path variable"); - Self::load_globals_from_config_str(&config, &mut globals)?; + + if globals.get(&FILE_PATH_VAR.into()).is_some() { + globals + .add(FILE_PATH_VAR.into(), BUILTINS_FILENAME.into()) + .expect("failed to add file path variable"); + } + sgl.build_stack_graph_into(graph, file, &source, &globals, cancellation_flag) .map_err(|err| LoadError::Builtins { inner: err, From a297647992b4a2f37e4aa11a3411039849d80a65 Mon Sep 17 00:00:00 2001 From: Hendrik van Antwerpen Date: Sun, 2 Jun 2024 11:31:51 +0200 Subject: [PATCH 7/9] Invert condition --- tree-sitter-stack-graphs/src/loader.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tree-sitter-stack-graphs/src/loader.rs b/tree-sitter-stack-graphs/src/loader.rs index d382f72c9..d03be0f93 100644 --- a/tree-sitter-stack-graphs/src/loader.rs +++ b/tree-sitter-stack-graphs/src/loader.rs @@ -83,7 +83,7 @@ impl LanguageConfiguration { Loader::load_globals_from_config_str(builtins_config, &mut builtins_globals)?; } - if builtins_globals.get(&FILE_PATH_VAR.into()).is_some() { + if builtins_globals.get(&FILE_PATH_VAR.into()).is_none() { builtins_globals .add(FILE_PATH_VAR.into(), BUILTINS_FILENAME.into()) .expect("failed to add file path variable"); From 6bcb5a3c3e39ed16db9ceaf2ae3eac03f4e337ce Mon Sep 17 00:00:00 2001 From: Hendrik van Antwerpen Date: Sun, 2 Jun 2024 11:32:08 +0200 Subject: [PATCH 8/9] Invery condition --- tree-sitter-stack-graphs/src/loader.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tree-sitter-stack-graphs/src/loader.rs b/tree-sitter-stack-graphs/src/loader.rs index d03be0f93..977522528 100644 --- a/tree-sitter-stack-graphs/src/loader.rs +++ b/tree-sitter-stack-graphs/src/loader.rs @@ -342,7 +342,7 @@ impl Loader { Self::load_globals_from_config_str(&config, &mut globals)?; - if globals.get(&FILE_PATH_VAR.into()).is_some() { + if globals.get(&FILE_PATH_VAR.into()).is_none() { globals .add(FILE_PATH_VAR.into(), BUILTINS_FILENAME.into()) .expect("failed to add file path variable"); From c7f9fd9118b0642ae800d4c42237be577afa7d41 Mon Sep 17 00:00:00 2001 From: nohehf Date: Sun, 2 Jun 2024 16:34:44 +0200 Subject: [PATCH 9/9] fix: check that FILE_PATH variable is none everywhere --- tree-sitter-stack-graphs/src/cli/test.rs | 18 ++++++++++-------- tree-sitter-stack-graphs/tests/it/test.rs | 17 ++++++++++------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/tree-sitter-stack-graphs/src/cli/test.rs b/tree-sitter-stack-graphs/src/cli/test.rs index 6ae04114c..790a6c245 100644 --- a/tree-sitter-stack-graphs/src/cli/test.rs +++ b/tree-sitter-stack-graphs/src/cli/test.rs @@ -318,15 +318,17 @@ impl TestArgs { )? { globals.clear(); - // Add FILE_PATH to variables - globals - .add( - FILE_PATH_VAR.into(), - test_fragment.path.to_str().unwrap().into(), - ) - .expect("failed to add file path variable"); - test_fragment.add_globals_to(&mut globals); + + if globals.get(&FILE_PATH_VAR.into()).is_none() { + globals + .add( + FILE_PATH_VAR.into(), + test_fragment.path.to_str().unwrap().into(), + ) + .expect("failed to add file path variable"); + } + lc.sgl.build_stack_graph_into( &mut test.graph, test_fragment.file, diff --git a/tree-sitter-stack-graphs/tests/it/test.rs b/tree-sitter-stack-graphs/tests/it/test.rs index 9624439c5..4c630d57a 100644 --- a/tree-sitter-stack-graphs/tests/it/test.rs +++ b/tree-sitter-stack-graphs/tests/it/test.rs @@ -100,14 +100,17 @@ fn check_test( for fragments in &test.fragments { globals.clear(); - globals - .add( - FILE_PATH_VAR.into(), - fragments.path.to_str().unwrap().into(), - ) - .expect("failed to add file path variable"); - fragments.add_globals_to(&mut globals); + + if globals.get(&FILE_PATH_VAR.into()).is_none() { + globals + .add( + FILE_PATH_VAR.into(), + fragments.path.to_str().unwrap().into(), + ) + .expect("failed to add file path variable"); + } + build_stack_graph_into( &mut test.graph, fragments.file,