Merge pull request #139 from Privado-Inc/jsSkipFile

consider exclusion rule in ast gen

joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/AstGenRunner.scala

@@ -27,6 +27,8 @@ object AstGenRunner {
 
   private val MinifiedPathRegex: Regex = ".*([.-]min\\..*js|bundle\\.js)".r
 
+  private val Extensions = Set(".js", ".ts", ".vue", ".ejs", ".jsx", ".cjs", ".mjs", ".tsx")
+
   private val AstGenDefaultIgnoreRegex: Seq[Regex] =
     List(
       "(conf|test|spec|[.-]min|\\.d)\\.(js|ts|jsx|tsx)$".r,
@@ -346,21 +348,18 @@ class AstGenRunner(config: Config) {
     logger.info(s"Parsed $numOfParsedFiles files.")
     if (numOfParsedFiles == 0) {
       logger.warn("You may want to check the DEBUG logs for a list of files that are ignored by default.")
-      SourceFiles.determine(
-        in.pathAsString,
-        Set(".js", ".ts", ".vue", ".ejs", ".jsx", ".cjs", ".mjs", ".tsx"),
-        ignoredDefaultRegex = Option(AstGenDefaultIgnoreRegex)
-      )
+      SourceFiles.determine(in.pathAsString, Extensions, ignoredDefaultRegex = Option(AstGenDefaultIgnoreRegex))
     }
     files
   }
 
   def execute(out: File): AstGenRunnerResult = {
-    val in = File(config.inputPath)
-    logger.info(s"Running astgen in '$in' ...")
+    val tmpInput = filterAndCopyFiles()
+    val in       = File(config.inputPath)
+    logger.info(s"Running astgen in '$tmpInput' ...")
     runAstGenNative(in, out) match {
       case Success(result) =>
-        val parsed  = checkParsedFiles(filterFiles(SourceFiles.determine(out.toString(), Set(".json")), out), in)
+        val parsed  = checkParsedFiles(filterFiles(SourceFiles.determine(out.toString(), Set(".json")), out), tmpInput)
         val skipped = skippedFiles(result.toList)
         AstGenRunnerResult(parsed.map((in.toString(), _)), skipped.map((in.toString(), _)))
       case Failure(f) =>
@@ -371,4 +370,26 @@ class AstGenRunner(config: Config) {
     }
   }
 
+  def filterAndCopyFiles(): File = {
+
+    /** Before running AstGen, filter and copy all the files in a temporary folder, which can be given as in input to
+      * AstGen, Earlier the filter used to happen post AstGen result, now it will be before. This helps in parsing files
+      * which are needed in AstGen
+      */
+    val filteredFiles = SourceFiles.determine(
+      config.inputPath,
+      Extensions,
+      ignoredDefaultRegex = Option(AstGenDefaultIgnoreRegex),
+      ignoredFilesRegex = Option(config.ignoredFilesRegex)
+    )
+    val tmpInput = File.newTemporaryDirectory("privadoGeneratedInput")
+    filteredFiles.foreach { filePath =>
+      val file            = File(filePath)
+      val destinationFile = tmpInput / Paths.get(config.inputPath).relativize(file.path).toString
+      destinationFile.parent.createDirectoryIfNotExists(createParents = true)
+      file.copyTo(destinationFile, overwrite = true)
+    }
+    tmpInput
+  }
+
 }

joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/preprocessing/AstGenTests.scala

@@ -0,0 +1,38 @@
+package io.joern.jssrc2cpg.preprocessing
+
+import better.files.File
+import io.joern.jssrc2cpg.Config
+import io.joern.jssrc2cpg.testfixtures.AstJsSrc2CpgSuite
+import io.joern.jssrc2cpg.utils.AstGenRunner
+
+class AstGenTests extends AstJsSrc2CpgSuite {
+
+  "Ast gen" should {
+    val tmpDir = File.newTemporaryDirectory("src")
+    ((tmpDir / "folder1").createDirectoryIfNotExists() / "1.js").write("console.log('folder1');")
+    ((tmpDir / "folder2").createDirectoryIfNotExists() / "1.js").write("console.log('folder2');")
+    ((tmpDir / "folder3").createDirectoryIfNotExists() / "1.js").write("console.log('folder3');")
+    "ignore files mentioned in exclusion regex" in {
+      val newInputDir = new AstGenRunner(Config().withInputPath(tmpDir.toString).withIgnoredFilesRegex(".*folder3.*"))
+        .filterAndCopyFiles()
+
+      val fileSet = newInputDir.listRecursively.filter(_.isRegularFile).map(_.pathAsString).toSet
+      fileSet.size shouldBe 2
+      fileSet.count(_.matches(".*folder1.*")) shouldBe 1
+      fileSet.count(_.matches(".*folder2.*")) shouldBe 1
+      fileSet.count(_.matches(".*folder3.*")) shouldBe 0
+    }
+
+    "don't ignore files, if exclusion regex not passed" in {
+      val newInputDir = new AstGenRunner(Config().withInputPath(tmpDir.toString)).filterAndCopyFiles()
+
+      val fileSet = newInputDir.listRecursively.filter(_.isRegularFile).map(_.pathAsString).toSet
+      fileSet.size shouldBe 3
+      fileSet.count(_.matches(".*folder1.*")) shouldBe 1
+      fileSet.count(_.matches(".*folder2.*")) shouldBe 1
+      fileSet.count(_.matches(".*folder3.*")) shouldBe 1
+
+    }
+  }
+
+}