Bug #22487 » clamp-malloc-increase.patch
| gc/default/default.c | ||
|---|---|---|
|
gc_counter_t malloc;
|
||
|
gc_counter_t free;
|
||
|
/* Baselines the increase is measured from: malloc snapshotted at GC start
|
||
|
* (gc_reset_malloc_info), free re-snapshotted at gc_sweep_finish so the
|
||
|
* sweep's own frees never count. */
|
||
|
/* Baselines snapshotted at GC start (gc_reset_malloc_info). */
|
||
|
gc_counter_t malloc_at_last_gc;
|
||
|
gc_counter_t free_at_last_gc;
|
||
|
};
|
||
| ... | ... | |
|
typedef struct rb_objspace {
|
||
|
struct {
|
||
|
struct gc_malloc_bytes counters;
|
||
|
size_t increase;
|
||
|
size_t mutator_increase;
|
||
|
#if RGENGC_ESTIMATE_OLDMALLOC
|
||
|
size_t oldmalloc_increase;
|
||
|
gc_counter_t freed_in_gc;
|
||
| ... | ... | |
|
}
|
||
|
#define malloc_limit objspace->malloc_params.limit
|
||
|
#define malloc_increase gc_malloc_counters_increase_unsigned(objspace, &objspace->malloc_counters.counters)
|
||
|
#define malloc_increase gc_malloc_increase_load(objspace)
|
||
|
#define malloc_allocated_size objspace->malloc_params.allocated_size
|
||
|
#ifdef MALLOC_COUNTERS_NEED_LOCK
|
||
| ... | ... | |
|
#endif
|
||
|
}
|
||
|
static inline int64_t
|
||
|
gc_malloc_counters_increase(rb_objspace_t *objspace, const struct gc_malloc_bytes *c)
|
||
|
{
|
||
|
MALLOC_COUNTERS_LOCK(objspace);
|
||
|
gc_counter_t malloc_at = gc_counter_load_acquire(&c->malloc_at_last_gc);
|
||
|
gc_counter_t free_at = gc_counter_load_acquire(&c->free_at_last_gc);
|
||
|
gc_counter_t malloc_now = gc_counter_load_relaxed(&c->malloc);
|
||
|
gc_counter_t free_now = gc_counter_load_relaxed(&c->free);
|
||
|
MALLOC_COUNTERS_UNLOCK(objspace);
|
||
|
gc_counter_t malloc_delta = malloc_now - malloc_at;
|
||
|
gc_counter_t free_delta = free_now - free_at;
|
||
|
if (malloc_delta >= free_delta) {
|
||
|
return (int64_t)(malloc_delta - free_delta);
|
||
|
}
|
||
|
else {
|
||
|
return -(int64_t)(free_delta - malloc_delta);
|
||
|
}
|
||
|
}
|
||
|
static inline size_t
|
||
|
gc_malloc_counters_increase_unsigned(rb_objspace_t *objspace, const struct gc_malloc_bytes *c)
|
||
|
gc_malloc_increase_load(rb_objspace_t *objspace)
|
||
|
{
|
||
|
int64_t inc = gc_malloc_counters_increase(objspace, c);
|
||
|
if (inc <= 0) return 0;
|
||
|
#if SIZEOF_SIZE_T < 8
|
||
|
if ((uint64_t)inc > SIZE_MAX) return SIZE_MAX;
|
||
|
#endif
|
||
|
return (size_t)inc;
|
||
|
return rbimpl_atomic_size_load(&objspace->malloc_counters.increase, RBIMPL_ATOMIC_RELAXED);
|
||
|
}
|
||
|
/* Frees done while sweeping are the GC's own work, not the mutator's: advance
|
||
|
* free_at_last_gc past them so they cannot pay for the next cycle's allocation.
|
||
|
* malloc_at_last_gc stays at gc_reset_malloc_info's snapshot (GC start). */
|
||
|
/* Frees done while sweeping are the GC's own work, not the mutator's: keep only the
|
||
|
* mutator's own allocations and frees since GC start, so the sweep's frees cannot
|
||
|
* pay for the next cycle's allocation. */
|
||
|
static inline void
|
||
|
gc_malloc_counters_snapshot_free_at_last_gc(rb_objspace_t *objspace, struct gc_malloc_bytes *c)
|
||
|
gc_malloc_increase_drop_sweep_frees(rb_objspace_t *objspace)
|
||
|
{
|
||
|
MALLOC_COUNTERS_LOCK(objspace);
|
||
|
gc_counter_store_release(&c->free_at_last_gc, gc_counter_load_relaxed(&c->free));
|
||
|
MALLOC_COUNTERS_UNLOCK(objspace);
|
||
|
size_t mutator_increase = rbimpl_atomic_size_load(&objspace->malloc_counters.mutator_increase, RBIMPL_ATOMIC_RELAXED);
|
||
|
RUBY_ATOMIC_SIZE_EXCHANGE(objspace->malloc_counters.increase, mutator_increase);
|
||
|
}
|
||
|
static inline void
|
||
| ... | ... | |
|
/* Not before: while sweeping is in progress its frees must keep reducing
|
||
|
* malloc_increase (objspace_malloc_increase_body sweeps and retries on it). */
|
||
|
gc_malloc_counters_snapshot_free_at_last_gc(objspace, &objspace->malloc_counters.counters);
|
||
|
gc_malloc_increase_drop_sweep_frees(objspace);
|
||
|
/* Leftovers from an earlier multi-Ractor phase: no later sweep can rediscover them
|
||
|
* (their slots are gone), and with one Ractor left the drain's barrier has nothing
|
||
| ... | ... | |
|
.malloc_at_last_gc = gc_counter_load_relaxed(&objspace->malloc_counters.counters.malloc_at_last_gc),
|
||
|
.free_at_last_gc = gc_counter_load_relaxed(&objspace->malloc_counters.counters.free_at_last_gc),
|
||
|
};
|
||
|
size_t saved_increase = gc_malloc_increase_load(objspace);
|
||
|
size_t saved_mutator_increase = rbimpl_atomic_size_load(&objspace->malloc_counters.mutator_increase, RBIMPL_ATOMIC_RELAXED);
|
||
|
#if RGENGC_ESTIMATE_OLDMALLOC
|
||
|
size_t saved_oldmalloc_increase = gc_oldmalloc_increase_load(objspace);
|
||
|
gc_counter_t saved_freed_in_gc = gc_counter_load_relaxed(&objspace->malloc_counters.freed_in_gc);
|
||
| ... | ... | |
|
gc_counter_store_release(&objspace->malloc_counters.counters.free, saved_malloc.free);
|
||
|
gc_counter_store_release(&objspace->malloc_counters.counters.malloc_at_last_gc, saved_malloc.malloc_at_last_gc);
|
||
|
gc_counter_store_release(&objspace->malloc_counters.counters.free_at_last_gc, saved_malloc.free_at_last_gc);
|
||
|
RUBY_ATOMIC_SIZE_EXCHANGE(objspace->malloc_counters.increase, saved_increase);
|
||
|
RUBY_ATOMIC_SIZE_EXCHANGE(objspace->malloc_counters.mutator_increase, saved_mutator_increase);
|
||
|
#if RGENGC_ESTIMATE_OLDMALLOC
|
||
|
RUBY_ATOMIC_SIZE_EXCHANGE(objspace->malloc_counters.oldmalloc_increase, saved_oldmalloc_increase);
|
||
|
gc_counter_store_release(&objspace->malloc_counters.freed_in_gc, saved_freed_in_gc);
|
||
| ... | ... | |
|
{
|
||
|
gc_prof_set_malloc_info(objspace);
|
||
|
{
|
||
|
int64_t inc = gc_malloc_counters_increase(objspace, &objspace->malloc_counters.counters);
|
||
|
size_t inc = RUBY_ATOMIC_SIZE_EXCHANGE(objspace->malloc_counters.increase, 0);
|
||
|
RUBY_ATOMIC_SIZE_EXCHANGE(objspace->malloc_counters.mutator_increase, 0);
|
||
|
gc_malloc_counters_snapshot(objspace, &objspace->malloc_counters.counters);
|
||
|
size_t old_limit = malloc_limit;
|
||
|
/* A net-negative `inc` (more freed than malloc'd since last GC) is
|
||
|
* treated the same as "allocated less than malloc_limit".
|
||
|
* This matches what we were doing pre-monotonic counters, but is it right? */
|
||
|
if (inc > 0 && (size_t)inc > malloc_limit) {
|
||
|
malloc_limit = (size_t)((size_t)inc * gc_params.malloc_limit_growth_factor);
|
||
|
if (inc > malloc_limit) {
|
||
|
malloc_limit = (size_t)(inc * gc_params.malloc_limit_growth_factor);
|
||
|
if (malloc_limit > gc_params.malloc_limit_max) {
|
||
|
malloc_limit = gc_params.malloc_limit_max;
|
||
|
}
|
||
| ... | ... | |
|
}
|
||
|
/* A global GC collects every objspace, so each needs the malloc-counter reset the
|
||
|
* driver got in gc_start_record; without it, gc_sweep_finish advancing free_at_last_gc
|
||
|
* (step 9) would leave their malloc_increase overstated by everything swept here. */
|
||
|
* driver got in gc_start_record; without it, gc_sweep_finish (step 9) would set their
|
||
|
* malloc_increase from a mutator_increase carried over from before this GC. */
|
||
|
for (size_t i = 0; i < global_objspace->global_gc.n_objspaces; i++) {
|
||
|
rb_objspace_t *const os = global_objspace->global_gc.objspaces[i];
|
||
|
if (os != driver) {
|
||
| ... | ... | |
|
* charged to dst, so without this transfer dst underestimates its own heap and delays
|
||
|
* GCs. dst is live, so take its counter lock where gc_counter_add is not atomic. */
|
||
|
{
|
||
|
int64_t inc = gc_malloc_counters_increase(src, &src->malloc_counters.counters);
|
||
|
size_t inc = gc_malloc_increase_load(src);
|
||
|
#if RGENGC_ESTIMATE_OLDMALLOC
|
||
|
size_t oldmalloc_increase = gc_oldmalloc_increase_load(src);
|
||
|
#endif
|
||
|
MALLOC_COUNTERS_LOCK(dst);
|
||
|
if (inc > 0) gc_counter_add(&dst->malloc_counters.counters.malloc, (size_t)inc);
|
||
|
if (inc > 0) {
|
||
|
gc_counter_add(&dst->malloc_counters.counters.malloc, inc);
|
||
|
RUBY_ATOMIC_SIZE_ADD(dst->malloc_counters.increase, inc);
|
||
|
RUBY_ATOMIC_SIZE_ADD(dst->malloc_counters.mutator_increase, inc);
|
||
|
}
|
||
|
#if RGENGC_ESTIMATE_OLDMALLOC
|
||
|
if (oldmalloc_increase > 0) RUBY_ATOMIC_SIZE_ADD(dst->malloc_counters.oldmalloc_increase, oldmalloc_increase);
|
||
|
#endif
|
||
| ... | ... | |
|
SET(heap_eden_pages, heap_eden_total_pages(objspace));
|
||
|
SET(total_allocated_pages, objspace->heap_pages.allocated_pages);
|
||
|
SET(total_freed_pages, objspace->heap_pages.freed_pages);
|
||
|
SET(malloc_increase_bytes, gc_malloc_counters_increase_unsigned(objspace, &objspace->malloc_counters.counters));
|
||
|
SET(malloc_increase_bytes, gc_malloc_increase_load(objspace));
|
||
|
SET(malloc_increase_bytes_limit, malloc_limit);
|
||
|
SET(minor_gc_count, objspace->profile.minor_gc_count);
|
||
|
SET(major_gc_count, objspace->profile.major_gc_count);
|
||
| ... | ... | |
|
size_t delta = new_size - old_size;
|
||
|
MALLOC_COUNTERS_LOCK(objspace);
|
||
|
gc_counter_add(&objspace->malloc_counters.counters.malloc, delta);
|
||
|
RUBY_ATOMIC_SIZE_ADD(objspace->malloc_counters.increase, delta);
|
||
|
RUBY_ATOMIC_SIZE_ADD(objspace->malloc_counters.mutator_increase, delta);
|
||
|
#if RGENGC_ESTIMATE_OLDMALLOC
|
||
|
RUBY_ATOMIC_SIZE_ADD(objspace->malloc_counters.oldmalloc_increase, delta);
|
||
|
#endif
|
||
| ... | ... | |
|
size_t delta = old_size - new_size;
|
||
|
MALLOC_COUNTERS_LOCK(objspace);
|
||
|
gc_counter_add(&objspace->malloc_counters.counters.free, delta);
|
||
|
atomic_sub_nounderflow(&objspace->malloc_counters.increase, delta);
|
||
|
if (!during_gc) atomic_sub_nounderflow(&objspace->malloc_counters.mutator_increase, delta);
|
||
|
#if RGENGC_ESTIMATE_OLDMALLOC
|
||
|
if (during_gc) gc_counter_add(&objspace->malloc_counters.freed_in_gc, delta);
|
||
|
atomic_sub_nounderflow(&objspace->malloc_counters.oldmalloc_increase, delta);
|
||
| test/ruby/test_gc.rb | ||
|---|---|---|
|
RUBY
|
||
|
end
|
||
|
def test_frees_do_not_delay_the_next_malloc_gc
|
||
|
omit 'default GC only' unless GC.config[:implementation] == 'default'
|
||
|
omit 'stress' if GC.stress
|
||
|
env = {
|
||
|
"RUBY_GC_MALLOC_LIMIT" => "16777216",
|
||
|
"RUBY_GC_MALLOC_LIMIT_MAX" => "16777216",
|
||
|
}
|
||
|
assert_separately([env, "-W0"], __FILE__, __LINE__, <<~'RUBY', timeout: 60)
|
||
|
buffers = Array.new(64) { "x" * (1 << 20) }
|
||
|
GC.start
|
||
|
buffers.each(&:clear)
|
||
|
count = GC.count
|
||
|
64.times { "y" * (1 << 20) }
|
||
|
assert_operator GC.count - count, :>=, 2
|
||
|
RUBY
|
||
|
end
|
||
|
def test_unfinished_lazy_sweep_does_not_delay_the_next_malloc_gc
|
||
|
omit 'default GC only' unless GC.config[:implementation] == 'default'
|
||
|
omit 'stress' if GC.stress
|
||
|
omit 'needs lazy sweep' unless GC::OPTS.include?('GC_ENABLE_LAZY_SWEEP')
|
||
|
env = {
|
||
|
"RUBY_GC_MALLOC_LIMIT" => "16777216",
|
||
|
"RUBY_GC_MALLOC_LIMIT_MAX" => "16777216",
|
||
|
}
|
||
|
assert_separately([env, "-W0"], __FILE__, __LINE__, <<~'RUBY', timeout: 60)
|
||
|
strings = Array.new(20_000) { String.new(capacity: 4096) }
|
||
|
kept = strings.each_slice(2).map(&:first)
|
||
|
strings = nil
|
||
|
GC.start
|
||
|
count = GC.count
|
||
|
256.times { String.new(capacity: 1 << 20) << "x" }
|
||
|
assert_operator GC.count - count, :>=, 12
|
||
|
assert_equal 10_000, kept.length
|
||
|
RUBY
|
||
|
end
|
||
|
def test_freed_buffers_do_not_count_as_malloc_increase
|
||
|
omit 'default GC only' unless GC.config[:implementation] == 'default'
|
||
|
omit 'stress' if GC.stress
|
||
|
omit 'needs lazy sweep' unless GC::OPTS.include?('GC_ENABLE_LAZY_SWEEP')
|
||
|
env = {
|
||
|
"RUBY_GC_MALLOC_LIMIT" => "16777216",
|
||
|
"RUBY_GC_MALLOC_LIMIT_MAX" => "16777216",
|
||
|
}
|
||
|
assert_separately([env, "-W0"], __FILE__, __LINE__, <<~'RUBY', timeout: 60)
|
||
|
objects = Array.new(400_000) { Object.new }
|
||
|
kept = objects.each_slice(2).map(&:first)
|
||
|
objects = nil
|
||
|
GC.start
|
||
|
GC::Profiler.enable
|
||
|
10_000.times do |i|
|
||
|
String.new(capacity: 1 << 20).clear
|
||
|
50.times { [i] }
|
||
|
end
|
||
|
malloc_gcs = GC::Profiler.raw_data.count { |row| row[:GC_FLAGS][:gc_by] == :malloc }
|
||
|
GC::Profiler.disable
|
||
|
assert_equal 0, malloc_gcs
|
||
|
assert_equal 200_000, kept.length
|
||
|
RUBY
|
||
|
end
|
||
|
def test_frees_during_lazy_sweep_do_not_cancel_later_allocations
|
||
|
omit 'default GC only' unless GC.config[:implementation] == 'default'
|
||
|
omit 'stress' if GC.stress
|
||
|
omit 'needs lazy sweep' unless GC::OPTS.include?('GC_ENABLE_LAZY_SWEEP')
|
||
|
env = {
|
||
|
"RUBY_GC_MALLOC_LIMIT" => "16777216",
|
||
|
"RUBY_GC_MALLOC_LIMIT_MAX" => "16777216",
|
||
|
}
|
||
|
assert_separately([env, "-W0"], __FILE__, __LINE__, <<~'RUBY', timeout: 60)
|
||
|
buffers = Array.new(64) { "x" * (1 << 20) }
|
||
|
GC.start
|
||
|
kept = []
|
||
|
200_000.times { |i| object = Object.new; kept << object if i.even? }
|
||
|
count = GC.count
|
||
|
1_000_000.times do
|
||
|
Object.new
|
||
|
break if GC.count != count && GC.latest_gc_info(:state) == :sweeping
|
||
|
end
|
||
|
assert_equal :sweeping, GC.latest_gc_info(:state)
|
||
|
buffers.each(&:clear)
|
||
|
count = GC.count
|
||
|
big = "y" * (48 << 20)
|
||
|
assert_operator GC.count - count, :>=, 1
|
||
|
assert_equal 48 << 20, big.bytesize
|
||
|
assert_equal 100_000, kept.length
|
||
|
RUBY
|
||
|
end
|
||
|
def test_stat_heap
|
||
|
omit 'stress' if GC.stress
|
||
