-
-
Notifications
You must be signed in to change notification settings - Fork 5.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' into prevent_stack_overflow_in_type_promotion
- Loading branch information
Showing
5 changed files
with
203 additions
and
94 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| // This file is a part of Julia. License is MIT: https://julialang.org/license | ||
|
|
||
| // ========================================================================= // | ||
| // Runtime Write-Barriers | ||
| // ========================================================================= // | ||
|
|
||
| #ifndef JL_GC_WB_H | ||
| #define JL_GC_WB_H | ||
|
|
||
| #ifdef __cplusplus | ||
| extern "C" { | ||
| #endif | ||
|
|
||
| extern void mmtk_object_reference_write_post(void* mutator, const void* parent, const void* ptr); | ||
| extern void mmtk_object_reference_write_slow(void* mutator, const void* parent, const void* ptr); | ||
| extern const void* MMTK_SIDE_LOG_BIT_BASE_ADDRESS; | ||
|
|
||
| #define MMTK_OBJECT_BARRIER (1) | ||
| // Stickyimmix needs write barrier. Immix does not need write barrier. | ||
| #ifdef MMTK_PLAN_IMMIX | ||
| #define MMTK_NEEDS_WRITE_BARRIER (0) | ||
| #endif | ||
| #ifdef MMTK_PLAN_STICKYIMMIX | ||
| #define MMTK_NEEDS_WRITE_BARRIER (1) | ||
| #endif | ||
|
|
||
| // Directly call into MMTk for write barrier (debugging only) | ||
| STATIC_INLINE void mmtk_gc_wb_full(const void *parent, const void *ptr) JL_NOTSAFEPOINT | ||
| { | ||
| jl_task_t *ct = jl_current_task; | ||
| jl_ptls_t ptls = ct->ptls; | ||
| mmtk_object_reference_write_post(&ptls->gc_tls.mmtk_mutator, parent, ptr); | ||
| } | ||
|
|
||
| // Inlined fastpath | ||
| STATIC_INLINE void mmtk_gc_wb_fast(const void *parent, const void *ptr) JL_NOTSAFEPOINT | ||
| { | ||
| if (MMTK_NEEDS_WRITE_BARRIER == MMTK_OBJECT_BARRIER) { | ||
| intptr_t addr = (intptr_t) (void*) parent; | ||
| uint8_t* meta_addr = (uint8_t*) (MMTK_SIDE_LOG_BIT_BASE_ADDRESS) + (addr >> 6); | ||
| intptr_t shift = (addr >> 3) & 0b111; | ||
| uint8_t byte_val = *meta_addr; | ||
| if (((byte_val >> shift) & 1) == 1) { | ||
| jl_task_t *ct = jl_current_task; | ||
| jl_ptls_t ptls = ct->ptls; | ||
| mmtk_object_reference_write_slow(&ptls->gc_tls.mmtk_mutator, parent, ptr); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| STATIC_INLINE void jl_gc_wb(const void *parent, const void *ptr) JL_NOTSAFEPOINT | ||
| { | ||
| mmtk_gc_wb_fast(parent, ptr); | ||
| } | ||
|
|
||
| STATIC_INLINE void jl_gc_wb_back(const void *ptr) JL_NOTSAFEPOINT // ptr isa jl_value_t* | ||
| { | ||
| mmtk_gc_wb_fast(ptr, (void*)0); | ||
| } | ||
|
|
||
| STATIC_INLINE void jl_gc_multi_wb(const void *parent, const jl_value_t *ptr) JL_NOTSAFEPOINT | ||
| { | ||
| mmtk_gc_wb_fast(parent, (void*)0); | ||
| } | ||
|
|
||
| STATIC_INLINE void jl_gc_wb_genericmemory_copy_boxed(const jl_value_t *dest_owner, _Atomic(void*) * dest_p, | ||
| jl_genericmemory_t *src, _Atomic(void*) * src_p, | ||
| size_t* n) JL_NOTSAFEPOINT | ||
| { | ||
| mmtk_gc_wb_fast(dest_owner, (void*)0); | ||
| } | ||
|
|
||
| STATIC_INLINE void jl_gc_wb_genericmemory_copy_ptr(const jl_value_t *owner, jl_genericmemory_t *src, char* src_p, | ||
| size_t n, jl_datatype_t *dt) JL_NOTSAFEPOINT | ||
| { | ||
| mmtk_gc_wb_fast(owner, (void*)0); | ||
| } | ||
|
|
||
|
|
||
| #ifdef __cplusplus | ||
| } | ||
| #endif | ||
|
|
||
| #endif |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| // This file is a part of Julia. License is MIT: https://julialang.org/license | ||
|
|
||
| // ========================================================================= // | ||
| // Runtime Write-Barriers | ||
| // ========================================================================= // | ||
|
|
||
| #ifndef JL_GC_WB_H | ||
| #define JL_GC_WB_H | ||
|
|
||
| #ifdef __cplusplus | ||
| extern "C" { | ||
| #endif | ||
|
|
||
| STATIC_INLINE void jl_gc_wb(const void *parent, const void *ptr) JL_NOTSAFEPOINT | ||
| { | ||
| // parent and ptr isa jl_value_t* | ||
| if (__unlikely(jl_astaggedvalue(parent)->bits.gc == 3 /* GC_OLD_MARKED */ && // parent is old and not in remset | ||
| (jl_astaggedvalue(ptr)->bits.gc & 1 /* GC_MARKED */) == 0)) // ptr is young | ||
| jl_gc_queue_root((jl_value_t*)parent); | ||
| } | ||
|
|
||
| STATIC_INLINE void jl_gc_wb_back(const void *ptr) JL_NOTSAFEPOINT // ptr isa jl_value_t* | ||
| { | ||
| // if ptr is old | ||
| if (__unlikely(jl_astaggedvalue(ptr)->bits.gc == 3 /* GC_OLD_MARKED */)) { | ||
| jl_gc_queue_root((jl_value_t*)ptr); | ||
| } | ||
| } | ||
|
|
||
| STATIC_INLINE void jl_gc_multi_wb(const void *parent, const jl_value_t *ptr) JL_NOTSAFEPOINT | ||
| { | ||
| // 3 == GC_OLD_MARKED | ||
| // ptr is an immutable object | ||
| if (__likely(jl_astaggedvalue(parent)->bits.gc != 3)) | ||
| return; // parent is young or in remset | ||
| if (__likely(jl_astaggedvalue(ptr)->bits.gc == 3)) | ||
| return; // ptr is old and not in remset (thus it does not point to young) | ||
| jl_datatype_t *dt = (jl_datatype_t*)jl_typeof(ptr); | ||
| const jl_datatype_layout_t *ly = dt->layout; | ||
| if (ly->npointers) | ||
| jl_gc_queue_multiroot((jl_value_t*)parent, ptr, dt); | ||
| } | ||
|
|
||
| STATIC_INLINE void jl_gc_wb_genericmemory_copy_boxed(const jl_value_t *dest_owner, _Atomic(void*) * dest_p, | ||
| jl_genericmemory_t *src, _Atomic(void*) * src_p, | ||
| size_t* n) JL_NOTSAFEPOINT | ||
| { | ||
| if (__unlikely(jl_astaggedvalue(dest_owner)->bits.gc == 3 /* GC_OLD_MARKED */ )) { | ||
| jl_value_t *src_owner = jl_genericmemory_owner(src); | ||
| size_t done = 0; | ||
| if (jl_astaggedvalue(src_owner)->bits.gc != 3 /* GC_OLD_MARKED */) { | ||
| if (dest_p < src_p || dest_p > src_p + (*n)) { | ||
| for (; done < (*n); done++) { // copy forwards | ||
| void *val = jl_atomic_load_relaxed(src_p + done); | ||
| jl_atomic_store_release(dest_p + done, val); | ||
| // `val` is young or old-unmarked | ||
| if (val && !(jl_astaggedvalue(val)->bits.gc & 1 /* GC_MARKED */)) { | ||
| jl_gc_queue_root(dest_owner); | ||
| break; | ||
| } | ||
| } | ||
| src_p += done; | ||
| dest_p += done; | ||
| } | ||
| else { | ||
| for (; done < (*n); done++) { // copy backwards | ||
| void *val = jl_atomic_load_relaxed(src_p + (*n) - done - 1); | ||
| jl_atomic_store_release(dest_p + (*n) - done - 1, val); | ||
| // `val` is young or old-unmarked | ||
| if (val && !(jl_astaggedvalue(val)->bits.gc & 1 /* GC_MARKED */)) { | ||
| jl_gc_queue_root(dest_owner); | ||
| break; | ||
| } | ||
| } | ||
| } | ||
| (*n) -= done; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| STATIC_INLINE void jl_gc_wb_genericmemory_copy_ptr(const jl_value_t *owner, jl_genericmemory_t *src, char* src_p, | ||
| size_t n, jl_datatype_t *dt) JL_NOTSAFEPOINT | ||
| { | ||
| if (__unlikely(jl_astaggedvalue(owner)->bits.gc == 3 /* GC_OLD_MARKED */)) { | ||
| jl_value_t *src_owner = jl_genericmemory_owner(src); | ||
| size_t elsz = dt->layout->size; | ||
| if (jl_astaggedvalue(src_owner)->bits.gc != 3 /* GC_OLD_MARKED */) { | ||
| dt = (jl_datatype_t*)jl_tparam1(dt); | ||
| for (size_t done = 0; done < n; done++) { // copy forwards | ||
| char* s = (char*)src_p+done*elsz; | ||
| if (*((jl_value_t**)s+dt->layout->first_ptr) != NULL) | ||
| jl_gc_queue_multiroot(owner, s, dt); | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| #ifdef __cplusplus | ||
| } | ||
| #endif | ||
|
|
||
| #endif |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters