feat(module): fixes module symbol conflicts in object files

compiler+runtime/include/cpp/jank/c_api.h

@@ -24,6 +24,8 @@ extern "C"
   jank_object_ptr jank_eval(jank_object_ptr s);
   jank_object_ptr jank_read_string(jank_object_ptr s);
 
+  void jank_ns_set_symbol_counter(char const * const ns, uint64_t count);
+
   jank_object_ptr jank_var_intern(jank_object_ptr ns, jank_object_ptr name);
   jank_object_ptr jank_var_bind_root(jank_object_ptr var, jank_object_ptr val);
   jank_object_ptr jank_var_set_dynamic(jank_object_ptr var, jank_object_ptr dynamic);

compiler+runtime/include/cpp/jank/jit/processor.hpp

@@ -34,6 +34,8 @@ namespace jank::jit
     void load_bitcode(native_persistent_string const &module,
                       native_persistent_string_view const &bitcode) const;
 
+    string_result<void> remove_symbol(native_persistent_string const &name) const;
+
     template <typename T>
     T find_symbol(native_persistent_string const &name) const
     {

compiler+runtime/include/cpp/jank/runtime/context.hpp

@@ -107,10 +107,10 @@ namespace jank::runtime
 
     /* Generates a unique name for use with anything from codgen structs,
      * lifted vars, to shadowed locals. */
-    static native_persistent_string unique_string();
-    static native_persistent_string unique_string(native_persistent_string_view const &prefix);
-    static obj::symbol unique_symbol();
-    static obj::symbol unique_symbol(native_persistent_string_view const &prefix);
+    native_persistent_string unique_string();
+    native_persistent_string unique_string(native_persistent_string_view const &prefix);
+    obj::symbol unique_symbol();
+    obj::symbol unique_symbol(native_persistent_string_view const &prefix);
 
     folly::Synchronized<native_unordered_map<obj::symbol_ptr, ns_ptr>> namespaces;
     folly::Synchronized<native_unordered_map<native_persistent_string, obj::keyword_ptr>> keywords;

compiler+runtime/include/cpp/jank/runtime/core/munge.hpp

@@ -1,10 +1,14 @@
 #pragma once
 
+#include <regex>
 #include <jank/runtime/object.hpp>
 
 namespace jank::runtime
 {
   native_persistent_string munge(native_persistent_string const &o);
+  native_persistent_string munge_extra(native_persistent_string const &o,
+                                       std::regex const &search,
+                                       char const * const replace);
   object_ptr munge(object_ptr o);
   native_persistent_string demunge(native_persistent_string const &o);
 }

compiler+runtime/include/cpp/jank/runtime/ns.hpp

@@ -50,6 +50,8 @@ namespace jank::runtime
     /* TODO: Benchmark the use of atomics here. That's what Clojure uses. */
     folly::Synchronized<obj::persistent_hash_map_ptr> vars;
     folly::Synchronized<obj::persistent_hash_map_ptr> aliases;
+
+    std::atomic_uint64_t symbol_counter{};
     context &rt_ctx;
   };
 }

compiler+runtime/src/cpp/clojure/core_native.cpp

@@ -78,6 +78,11 @@ namespace clojure::core_native
     return try_object<var>(o)->deref();
   }
 
+  static object_ptr intern_var(object_ptr const sym)
+  {
+    return __rt_ctx->intern_var(try_object<obj::symbol>(sym)).expect_ok();
+  }
+
   static object_ptr var_get_root(object_ptr const o)
   {
     return try_object<var>(o)->get_root();
@@ -410,6 +415,7 @@ jank_object_ptr jank_load_clojure_core_native()
   intern_fn("namespace", &namespace_);
   intern_fn("var?", &core_native::is_var);
   intern_fn("var-get", &core_native::var_get);
+  intern_fn("intern-var", &core_native::intern_var);
   intern_fn("var-get-root", &core_native::var_get_root);
   intern_fn("var-bind-root", &core_native::var_bind_root);
   intern_fn("alter-var-root", &core_native::alter_var_root);

compiler+runtime/src/cpp/jank/analyze/local_frame.cpp

@@ -248,7 +248,7 @@ namespace jank::analyze
       return;
     }
 
-    auto const name(context::unique_symbol("const"));
+    auto const name(__rt_ctx->unique_symbol("const"));
     auto const unboxed_name{ visit_number_like(
       [&](auto const) -> option<obj::symbol> {
         return obj::symbol{ name.ns, name.name + "__unboxed" };

compiler+runtime/src/cpp/jank/analyze/processor.cpp

@@ -312,7 +312,7 @@ namespace jank::analyze
           {
             /* C++ doesn't allow multiple params with the same name, so we generate a unique
              * name for shared params. */
-            param = make_box<runtime::obj::symbol>(runtime::context::unique_string("shadowed"));
+            param = make_box<runtime::obj::symbol>(__rt_ctx->unique_string("shadowed"));
             break;
           }
         }
@@ -387,7 +387,7 @@ namespace jank::analyze
     {
       auto const s(runtime::expect_object<runtime::obj::symbol>(first_elem));
       name = s->name;
-      unique_name = runtime::context::unique_string(name);
+      unique_name = __rt_ctx->unique_string(name);
       if(length < 3)
       {
         return err(error{ fmt::format("fn missing forms: {}", full_list->to_string()) });
@@ -397,7 +397,7 @@ namespace jank::analyze
     }
     else
     {
-      name = runtime::context::unique_string("fn");
+      name = __rt_ctx->unique_string("fn");
       unique_name = name;
     }
 

compiler+runtime/src/cpp/jank/c_api.cpp

@@ -75,9 +75,17 @@ extern "C"
     return __rt_ctx->read_string(s_obj->data);
   }
 
+  void jank_ns_set_symbol_counter(char const * const ns, uint64_t count)
+  {
+    auto const ns_obj(__rt_ctx->intern_ns(ns));
+    ns_obj->symbol_counter.store(count);
+  }
+
   jank_object_ptr jank_var_intern(jank_object_ptr const ns, jank_object_ptr const name)
   {
     auto const ns_obj(try_object<obj::persistent_string>(reinterpret_cast<object *>(ns)));
+    __rt_ctx->intern_ns(ns_obj->data);
+
     auto const name_obj(try_object<obj::persistent_string>(reinterpret_cast<object *>(name)));
     return erase(__rt_ctx->intern_var(ns_obj->data, name_obj->data).expect_ok());
   }

compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp

@@ -26,9 +26,9 @@ namespace jank::codegen
 
   reusable_context::reusable_context(native_persistent_string const &module_name)
     : module_name{ module_name }
-    , ctor_name{ runtime::munge(runtime::context::unique_string("jank_global_init")) }
+    , ctor_name{ runtime::munge(__rt_ctx->unique_string("jank_global_init")) }
     , llvm_ctx{ std::make_unique<llvm::LLVMContext>() }
-    , module{ std::make_unique<llvm::Module>(runtime::context::unique_string(module_name).c_str(),
+    , module{ std::make_unique<llvm::Module>(__rt_ctx->unique_string(module_name).c_str(),
                                              *llvm_ctx) }
     , builder{ std::make_unique<llvm::IRBuilder<>>(*llvm_ctx) }
     , global_ctor_block{ llvm::BasicBlock::Create(*llvm_ctx, "entry") }
@@ -135,6 +135,18 @@ namespace jank::codegen
     {
       auto const global_ctor_fn(ctx->global_ctor_block->getParent());
       ctx->builder->CreateCall(global_ctor_fn, {});
+
+      auto const current_ns{ __rt_ctx->current_ns() };
+      auto const fn_type(
+        llvm::FunctionType::get(ctx->builder->getVoidTy(),
+                                { ctx->builder->getPtrTy(), ctx->builder->getInt64Ty() },
+                                false));
+      auto const fn(ctx->module->getOrInsertFunction("jank_ns_set_symbol_counter", fn_type));
+
+      ctx->builder->CreateCall(
+        fn,
+        { gen_c_string(current_ns->name->get_name()),
+          llvm::ConstantInt::get(ctx->builder->getInt64Ty(), current_ns->symbol_counter.load()) });
     }
 
     for(size_t i{}; i < arity.params.size(); ++i)

compiler+runtime/src/cpp/jank/evaluate.cpp

@@ -134,7 +134,7 @@ namespace jank::evaluate
     auto &fn(boost::get<expr::function<expression>>(ret->data));
     expr::function_arity<expression> arity;
     fn.name = name;
-    fn.unique_name = context::unique_string(fn.name);
+    fn.unique_name = __rt_ctx->unique_string(fn.name);
     fn.meta = obj::persistent_hash_map::empty();
 
     auto const &closest_fn_frame(local_frame::find_closest_fn_frame(*expr.frame));

compiler+runtime/src/cpp/jank/jit/processor.cpp

@@ -4,6 +4,7 @@
 #include <clang/Basic/Diagnostic.h>
 #include <clang/Frontend/CompilerInstance.h>
 #include <clang/Frontend/FrontendDiagnostic.h>
+#include <llvm/ExecutionEngine/Orc/Core.h>
 #include <llvm/Support/Signals.h>
 #include <llvm/ExecutionEngine/Orc/LLJIT.h>
 #include <llvm/IRReader/IRReader.h>
@@ -185,6 +186,20 @@ namespace jank::jit
     load_ir_module(std::move(ir_module), std::move(ctx));
   }
 
+  string_result<void> processor::remove_symbol(native_persistent_string const &name) const
+  {
+    auto &ee{ interpreter->getExecutionEngine().get() };
+    llvm::orc::SymbolNameSet to_remove{};
+    to_remove.insert(ee.getExecutionSession().intern(name.c_str()));
+
+    if(auto error = ee.getMainJITDylib().remove(to_remove);
+       !error.isA<llvm::orc::SymbolsNotFound>())
+    {
+      return err("Failed to remove the symbol");
+    }
+    return ok();
+  }
+
   option<native_persistent_string>
   processor::find_dynamic_lib(native_persistent_string const &lib) const
   {

compiler+runtime/src/cpp/jank/read/parse.cpp

@@ -914,7 +914,7 @@ namespace jank::read::parse
         auto gensym(get(env, sym));
         if(gensym->type == object_type::nil)
         {
-          gensym = make_box<obj::symbol>(context::unique_symbol(sym->name));
+          gensym = make_box<obj::symbol>(__rt_ctx->unique_symbol(sym->name));
           __rt_ctx->gensym_env_var->set(assoc(env, sym, gensym)).expect_ok();
         }
         sym = expect_object<obj::symbol>(gensym);

compiler+runtime/src/cpp/jank/runtime/context.cpp

@@ -8,13 +8,15 @@
 #include <llvm/TargetParser/Host.h>
 
 #include <fmt/compile.h>
+#include <regex>
 
 #include <jank/native_persistent_string/fmt.hpp>
 #include <jank/read/lex.hpp>
 #include <jank/read/parse.hpp>
 #include <jank/runtime/context.hpp>
 #include <jank/runtime/visit.hpp>
 #include <jank/runtime/core.hpp>
+#include <jank/runtime/core/munge.hpp>
 #include <jank/analyze/processor.hpp>
 #include <jank/evaluate.hpp>
 #include <jank/jit/processor.hpp>
@@ -264,7 +266,7 @@ namespace jank::runtime
                                     std::make_pair(compile_files_var, obj::boolean::true_const()),
                                     std::make_pair(current_module_var, make_box(module))) };
 
-    return load_module(fmt::format("/{}", module), module::origin::source);
+    return load_module(fmt::format("/{}", module), module::origin::latest);
   }
 
   string_result<void>
@@ -285,7 +287,7 @@ namespace jank::runtime
                              module_path.c_str(),
                              file_error.message()));
     }
-    //codegen_ctx->module->print(llvm::outs(), nullptr);
+    // codegen_ctx->module->print(llvm::outs(), nullptr);
 
     auto const target_triple{ llvm::sys::getDefaultTargetTriple() };
     std::string target_error;
@@ -321,8 +323,12 @@ namespace jank::runtime
 
   native_persistent_string context::unique_string(native_persistent_string_view const &prefix)
   {
-    static std::atomic_size_t index{ 1 };
-    return fmt::format(FMT_COMPILE("{}-{}"), prefix.data(), index++);
+    static std::regex const dot{ "\\." };
+    auto const ns{ current_ns() };
+    return fmt::format(FMT_COMPILE("{}-{}-{}"),
+                       runtime::munge_extra(ns->name->get_name(), dot, "_"),
+                       prefix.data(),
+                       ++ns->symbol_counter);
   }
 
   obj::symbol context::unique_symbol()

compiler+runtime/src/cpp/jank/runtime/core.cpp

@@ -448,7 +448,7 @@ namespace jank::runtime
 
   object_ptr gensym(object_ptr const o)
   {
-    return make_box<obj::symbol>(runtime::context::unique_symbol(to_string(o)));
+    return make_box<obj::symbol>(__rt_ctx->unique_symbol(to_string(o)));
   }
 
   object_ptr atom(object_ptr const o)

compiler+runtime/src/cpp/jank/runtime/core/munge.cpp

@@ -1,3 +1,5 @@
+#include <regex>
+
 #include <jank/runtime/core/munge.hpp>
 #include <jank/runtime/obj/persistent_string.hpp>
 #include <jank/runtime/rtti.hpp>
@@ -283,6 +285,14 @@ namespace jank::runtime
     return munged;
   }
 
+  native_persistent_string munge_extra(native_persistent_string const &o,
+                                       std::regex const &search,
+                                       char const * const replace)
+  {
+    std::string ret{ munge(o) };
+    return std::regex_replace(ret, search, replace);
+  }
+
   /* TODO: Support symbols and other data; Clojure takes in anything and passes it through str. */
   object_ptr munge(object_ptr const o)
   {

compiler+runtime/src/cpp/jank/runtime/module/loader.cpp

@@ -39,19 +39,13 @@ namespace jank::runtime::module
   native_persistent_string module_to_path(native_persistent_string_view const &module)
   {
     static std::regex const dot{ "\\." };
-
-    std::string ret{ runtime::munge(module) };
-    ret = std::regex_replace(ret, dot, "/");
-
-    return ret;
+    return runtime::munge_extra(module, dot, "/");
   }
 
   native_persistent_string module_to_load_function(native_persistent_string_view const &module)
   {
     static std::regex const dot{ "\\." };
-
-    std::string ret{ runtime::munge(module) };
-    ret = std::regex_replace(ret, dot, "_");
+    std::string ret{ runtime::munge_extra(module, dot, "_") };
 
     return fmt::format("jank_load_{}", ret);
   }
@@ -469,6 +463,13 @@ namespace jank::runtime::module
   loader::load_o(native_persistent_string const &module, file_entry const &entry) const
   {
     profile::timer const timer{ fmt::format("load object {}", module) };
+    auto const res{ rt_ctx.jit_prc.remove_symbol(module_to_load_function(module)) };
+
+    if(res.is_err())
+    {
+      return res.expect_err();
+    }
+
     if(entry.archive_path.is_some())
     {
       /* TODO: Load object code from string. */

compiler+runtime/src/jank/clojure/core.jank

@@ -3638,7 +3638,7 @@
 
 (def find-ns
   "Returns the namespace named by the symbol or nil if it doesn't exist."
-  clojure.core-native/create-ns)
+  clojure.core-native/find-ns)
 
 (def remove-ns
   "Removes the namespace named by the symbol. Use with caution.
@@ -3992,13 +3992,32 @@
          ~@(when (and (not= name 'clojure.core) refer-full-clojure?)
              `((clojure.core/refer '~'clojure.core)))
          ~@(map process-reference references))
-       (if (= '~name 'clojure.core)
+       (if (= '~name '~'clojure.core)
           nil
-          (do (swap! *loaded-libs* conj '~name) nil)))))
+          (do (swap! @#'*loaded-libs* conj '~name) nil)))))
 
 (defmacro refer-clojure
   "Same as (refer 'clojure.core <filters>)"
   [& filters]
   `(clojure.core/refer '~'clojure.core ~@filters))
 
+(defn intern
+  "Finds or creates a var named by the symbol name in the namespace
+  ns (which can be a symbol or a namespace), setting its root binding
+  to val if supplied. The namespace must exist. The var will adopt any
+  metadata from the name symbol.  Returns the var."
+  ([ns name]
+     (let [v (clojure.core-native/intern-var
+               (clojure.core-native/->qualified-symbol
+                 (the-ns ns) name))]
+       (when (meta name) (clojure.core-native/reset-meta! v (meta name)))
+       v))
+  ([ns name val]
+     (let [v (clojure.core-native/intern-var
+               (clojure.core-native/->qualified-symbol
+                 (the-ns ns) name))]
+       (clojure.core-native/var-bind-root v val)
+       (when (meta name) (clojure.core-native/reset-meta! v (meta name)))
+       v)))
+
 (println "Bottom of clojure.core")