Bundle nested FFI on publish

compiler-cli/src/fs.rs

@@ -2,8 +2,8 @@ use gleam_core::{
     build::{NullTelemetry, Target},
     error::{parse_os, Error, FileIoAction, FileKind, ShellCommandFailureReason, OS},
     io::{
-        BeamCompiler, Command, CommandExecutor, Content, DirEntry, FileSystemReader,
-        FileSystemWriter, OutputFile, ReadDir, Stdio, WrappedReader,
+        is_native_file_extension, BeamCompiler, Command, CommandExecutor, Content, DirEntry,
+        FileSystemReader, FileSystemWriter, OutputFile, ReadDir, Stdio, WrappedReader,
     },
     language_server::{DownloadDependencies, Locker, MakeLocker},
     manifest::Manifest,
@@ -402,14 +402,20 @@ pub fn gleam_files_excluding_gitignore(dir: &Utf8Path) -> impl Iterator<Item = U
         .filter(move |d| is_gleam_path(d, dir))
 }
 
-pub fn native_files(dir: &Utf8Path) -> Result<impl Iterator<Item = Utf8PathBuf> + '_> {
-    Ok(read_dir(dir)?
-        .flat_map(Result::ok)
-        .map(|e| e.into_path())
+pub fn native_files(dir: &Utf8Path) -> impl Iterator<Item = Utf8PathBuf> + '_ {
+    ignore::WalkBuilder::new(dir)
+        .follow_links(true)
+        .require_git(false)
+        .filter_entry(|e| !is_gleam_build_dir(e))
+        .build()
+        .filter_map(Result::ok)
+        .filter(|e| e.file_type().map(|t| t.is_file()).unwrap_or(false))
+        .map(ignore::DirEntry::into_path)
+        .map(|pb| Utf8PathBuf::from_path_buf(pb).expect("Non Utf-8 Path"))
         .filter(|path| {
             let extension = path.extension().unwrap_or_default();
-            matches!(extension, "erl" | "hrl" | "ex" | "js" | "mjs" | "ts")
-        }))
+            is_native_file_extension(extension)
+        })
 }
 
 pub fn private_files_excluding_gitignore(dir: &Utf8Path) -> impl Iterator<Item = Utf8PathBuf> + '_ {
@@ -423,7 +429,7 @@ pub fn private_files_excluding_gitignore(dir: &Utf8Path) -> impl Iterator<Item =
         .map(|pb| Utf8PathBuf::from_path_buf(pb).expect("Non Utf-8 Path"))
 }
 
-pub fn erlang_files(dir: &Utf8Path) -> Result<impl Iterator<Item = Utf8PathBuf> + '_> {
+pub fn toplevel_erlang_files(dir: &Utf8Path) -> Result<impl Iterator<Item = Utf8PathBuf> + '_> {
     Ok(read_dir(dir)?
         .flat_map(Result::ok)
         .map(|e| e.into_path())

compiler-cli/src/publish.rs

@@ -346,7 +346,7 @@ fn do_build_hex_tarball(paths: &ProjectPaths, config: &mut PackageConfig) -> Res
         Target::Erlang => generated_erlang_files(paths, &built.root_package)?,
         Target::JavaScript => vec![],
     };
-    let src_files = project_files()?;
+    let src_files = project_files(Utf8Path::new(""))?;
     let contents_tar_gz = contents_tarball(&src_files, &generated_files)?;
     let version = "3";
     let metadata = metadata_config(&built.root_package.config, &src_files, &generated_files)?;
@@ -451,19 +451,17 @@ fn contents_tarball(
     Ok(contents_tar_gz)
 }
 
-// TODO: test
-// TODO: Don't include git-ignored native files
-fn project_files() -> Result<Vec<Utf8PathBuf>> {
-    let src = Utf8Path::new("src");
-    let mut files: Vec<Utf8PathBuf> = fs::gleam_files_excluding_gitignore(src)
-        .chain(fs::native_files(src)?)
+fn project_files(base_path: &Utf8Path) -> Result<Vec<Utf8PathBuf>> {
+    let src = base_path.join(Utf8Path::new("src"));
+    let mut files: Vec<Utf8PathBuf> = fs::gleam_files_excluding_gitignore(&src)
+        .chain(fs::native_files(&src))
         .collect();
-    let private = Utf8Path::new("priv");
+    let private = base_path.join(Utf8Path::new("priv"));
     let mut private_files: Vec<Utf8PathBuf> =
-        fs::private_files_excluding_gitignore(private).collect();
+        fs::private_files_excluding_gitignore(&private).collect();
     files.append(&mut private_files);
     let mut add = |path| {
-        let path = Utf8PathBuf::from(path);
+        let path = base_path.join(path);
         if path.exists() {
             files.push(path);
         }
@@ -510,7 +508,12 @@ fn generated_erlang_files(
 
     // Erlang headers
     if include.is_dir() {
-        for file in fs::erlang_files(&include)? {
+        // Note that the include subdirectory of 'build/prod/.../package'  is
+        // auto-generated on Erlang codegen based on exposed record types, and
+        // is always flat, so we can check only for top-level Erlang files.
+        // Precisely because it is auto-generated, we also don't check for
+        // '.gitignore' since it would typically always ignore 'build/' anyway.
+        for file in fs::toplevel_erlang_files(&include)? {
             let name = file.file_name().expect("generated_files include file name");
             files.push((tar_include.join(name), fs::read(file)?));
         }
@@ -754,3 +757,107 @@ fn prevent_publish_git_dependency() {
 fn quotes(x: &str) -> String {
     format!(r#"<<"{x}">>"#)
 }
+
+#[test]
+fn exported_project_files_test() {
+    let tmp = tempfile::tempdir().unwrap();
+    let path = Utf8PathBuf::from_path_buf(tmp.path().join("my_project")).expect("Non Utf8 Path");
+
+    let exported_project_files = &[
+        "LICENCE",
+        "LICENCE.md",
+        "LICENCE.txt",
+        "LICENSE",
+        "LICENSE.md",
+        "LICENSE.txt",
+        "NOTICE",
+        "NOTICE.md",
+        "NOTICE.txt",
+        "README",
+        "README.md",
+        "README.txt",
+        "gleam.toml",
+        "priv/wibble",
+        "priv/wobble.js",
+        "src/exported.gleam",
+        "src/exported_ffi.erl",
+        "src/exported_ffi.ex",
+        "src/exported_ffi.hrl",
+        "src/exported_ffi.js",
+        "src/exported_ffi.mjs",
+        "src/exported_ffi.ts",
+        "src/nested/exported.gleam",
+        "src/nested/exported_ffi.erl",
+        "src/nested/exported_ffi.ex",
+        "src/nested/exported_ffi.hrl",
+        "src/nested/exported_ffi.js",
+        "src/nested/exported_ffi.mjs",
+        "src/nested/exported_ffi.ts",
+    ];
+
+    let unexported_project_files = &[
+        ".git/",
+        ".github/workflows/test.yml",
+        ".gitignore",
+        "build/",
+        "test/exported_test.gleam",
+        "test/exported_test_ffi.erl",
+        "test/exported_test_ffi.ex",
+        "test/exported_test_ffi.hrl",
+        "test/exported_test_ffi.js",
+        "test/exported_test_ffi.mjs",
+        "test/exported_test_ffi.ts",
+        "test/nested/exported_test.gleam",
+        "test/nested/exported_test_ffi.erl",
+        "test/nested/exported_test_ffi.ex",
+        "test/nested/exported_test_ffi.hrl",
+        "test/nested/exported_test_ffi.js",
+        "test/nested/exported_test_ffi.mjs",
+        "test/nested/exported_test_ffi.ts",
+    ];
+
+    let gitignored_files = &[
+        ".ignored.txt",
+        "priv/.ignored",
+        "src/.ignored.gleam",
+        "src/.ignored_ffi.mjs",
+        "src/.ignored_ffi.erl",
+        "src/nested/.ignored.gleam",
+        "src/nested/.ignored_ffi.erl",
+        "src/nested/.ignored_ffi.mjs",
+        "test/.ignored_test.gleam",
+        "test/.ignored_test_ffi.erl",
+        "test/.ignored_test_ffi.mjs",
+        "test/nested/.ignored.gleam",
+        "test/nested/.ignored_test_ffi.erl",
+        "test/nested/.ignored_test_ffi.mjs",
+    ];
+
+    for &file in exported_project_files
+        .iter()
+        .chain(unexported_project_files)
+        .chain(gitignored_files)
+    {
+        if file.ends_with("/") {
+            fs::mkdir(path.join(file)).unwrap();
+            continue;
+        }
+
+        let contents = match file {
+            ".gitignore" => gitignored_files.join("\n"),
+            _ => "".to_string(),
+        };
+
+        fs::write(&path.join(file), &contents).unwrap();
+    }
+
+    let mut chosen_exported_files = project_files(&path).unwrap();
+    chosen_exported_files.sort_unstable();
+
+    let expected_exported_files = exported_project_files
+        .iter()
+        .map(|s| path.join(s))
+        .collect_vec();
+
+    assert_eq!(expected_exported_files, chosen_exported_files);
+}

compiler-core/src/build/native_file_copier.rs

@@ -95,7 +95,7 @@ where
         }
 
         // Skip unknown file formats that are not supported native files
-        if !matches!(extension, "mjs" | "js" | "ts" | "hrl" | "erl" | "ex") {
+        if !crate::io::is_native_file_extension(extension) {
             return Ok(());
         }
 

compiler-core/src/io.rs

@@ -446,6 +446,11 @@ pub trait TarUnpacker {
     }
 }
 
+#[inline]
+pub fn is_native_file_extension(extension: &str) -> bool {
+    matches!(extension, "erl" | "hrl" | "ex" | "js" | "mjs" | "ts")
+}
+
 pub fn ordered_map<S, K, V>(value: &HashMap<K, V>, serializer: S) -> Result<S::Ok, S::Error>
 where
     S: serde::Serializer,