src/ex/recycling_memory_resource.cpp

100.0% Lines (101 / 101) 100.0% Functions (14 / 14)

[ ] prev/next file

recycling_memory_resource.cpp

f(x) Functions (14)

Function Calls Lines Blocks

Line	Hits	Source Code
1		//
2		// Copyright (c) 2025 Vinnie Falco (vinnie.falco@gmail.com)
3		//
4		// Distributed under the Boost Software License, Version 1.0. (See accompanying
5		// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6		//
7		// Official repository: https://github.com/cppalliance/capy
8		//
9
10		#include <boost/capy/ex/recycling_memory_resource.hpp>
11
12		#include <new>
13		#include <tuple>
14
15		namespace boost {
16		namespace capy {
17
18		// Instance destruction does nothing: the resource is stateless (all pools
19		// are static). Global-pool cleanup is owned by global()'s holder, exactly
20		// as in the original where the global pool's own destructor drained it,
21		// independent of any instance lifetime.
22	327x	recycling_memory_resource::~recycling_memory_resource() = default;
23
24		#if (defined(BOOST_CAPY_DYN_LINK) \|\| defined(BOOST_ALL_DYN_LINK)) && \
25		!defined(BOOST_CAPY_STATIC_LINK)
26
27		recycling_memory_resource::pool&
28	2590777x	recycling_memory_resource::local() noexcept
29		{
30		static thread_local pool p;
31	2590777x	return p;
32		}
33
34		bool&
35	1144341x	recycling_memory_resource::thread_armed() noexcept
36		{
37		static thread_local bool armed = false;
38	1144341x	return armed;
39		}
40
41		#endif
42
43		namespace {
44
45		// Set once this thread's pool was drained at exit. Another thread_local
46		// destroyed later can still free frames; they must not be cached.
47		thread_local bool pool_drained = false;
48
49		} // namespace
50
51		bool
52	302819x	recycling_memory_resource::arm_thread_cleanup() noexcept
53		{
54		struct janitor
55		{
56	373x	~janitor()
57		{
58		// Return this thread's cached blocks to the OS so nothing
59		// leaks at thread exit. The pool has a trivial dtor, so its
60		// thread-local storage is still valid here.
61	373x	auto& lp = local();
62	2611x	for(auto& b : lp.buckets)
63	52545x	while(b.count > 0)
64	50307x	::operator delete(b.pop());
65	373x	pool_drained = true;
66	373x	thread_armed() = false;
67	373x	}
68		};
69	302819x	if(pool_drained)
70	5x	return false;
71	302814x	static thread_local janitor j;
72	302814x	std::ignore = j;
73	302814x	return true;
74		}
75
76		recycling_memory_resource::pool&
77	569107x	recycling_memory_resource::global() noexcept
78		{
79		// Holder gives the global pool a destructor (the trivial-dtor pool type
80		// itself must stay guard-free for local()). Runs unconditionally at
81		// process exit, mirroring the original global pool destructor, and is
82		// locked because a worker thread may still be in a slow path. This path
83		// is cold (slow paths only), so the holder's guard costs nothing hot.
84		struct holder
85		{
86		pool p;
87
88	33x	~holder()
89		{
90	33x	std::lock_guard<std::mutex> lock(global_mutex());
91	231x	for(auto& b : p.buckets)
92	5320x	while(b.count > 0)
93	5122x	::operator delete(b.pop());
94	33x	}
95		};
96	569107x	static holder h;
97	569107x	return h.p;
98		}
99
100		std::mutex&
101	569140x	recycling_memory_resource::global_mutex() noexcept
102		{
103		// Never destroyed: it is locked during process-exit teardown (in
104		// global()'s holder destructor), after a function-local `static
105		// std::mutex` could itself have been destroyed. Placement-new into
106		// static storage owns no heap allocation, so there is nothing to leak.
107		alignas(std::mutex) static unsigned char storage[sizeof(std::mutex)];
108		static std::mutex* const mtx =
109	569140x	::new(static_cast<void*>(storage)) std::mutex();
110	569140x	return *mtx;
111		}
112
113		void*
114	302444x	recycling_memory_resource::allocate_slow(
115		std::size_t rounded, std::size_t idx)
116		{
117		// Once this thread's pool was drained at exit, take one block and
118		// leave the pool empty: nothing would release a refill.
119	302444x	bool const cache = arm_thread_cleanup();
120	302444x	void* p = nullptr;
121	302444x	std::size_t refilled = 0;
122		{
123	302444x	std::lock_guard<std::mutex> lock(global_mutex());
124	302444x	auto& b = global().buckets[idx];
125	302444x	if(cache)
126		{
127	302443x	auto& lb = local().buckets[idx];
128	302443x	p = b.pop(lb);
129	302443x	refilled = lb.count;
130		}
131		else
132		{
133	1x	p = b.pop();
134		}
135	302444x	}
136	302444x	if(p == nullptr)
137	302415x	return ::operator new(rounded);
138
139		// A thread that only allocates never frees into its pool, so its
140		// refills are what drive its decay passes.
141	29x	auto& lp = local();
142	29x	lp.arrivals += refilled;
143	29x	if(lp.arrivals >= decay_period)
144	7x	decay(lp);
145	29x	return p;
146		}
147
148		void
149	266384x	recycling_memory_resource::deallocate_slow(
150		void* p, std::size_t idx)
151		{
152		{
153	266384x	std::lock_guard<std::mutex> lock(global_mutex());
154	266384x	if(global().buckets[idx].push(p, bucket_capacity(idx)))
155	59059x	return;
156	266384x	}
157	207325x	::operator delete(p);
158		}
159
160		void
161	279x	recycling_memory_resource::decay(pool& lp) noexcept
162		{
163	279x	lp.arrivals = 0;
164
165		// Unlink each bucket's share before taking the lock, so the walk
166		// through the blocks happens outside it.
167	279x	node* heads[num_classes] = {};
168	279x	node* tails[num_classes] = {};
169	279x	std::size_t counts[num_classes] = {};
170	1953x	for(std::size_t i = 0; i < num_classes; ++i)
171		{
172	1674x	auto& b = lp.buckets[i];
173	1674x	std::size_t const share = (b.low + 1) / 2;
174	1674x	if(share > 0)
175		{
176	122x	heads[i] = b.head;
177	122x	node* t = b.head;
178	48938x	for(std::size_t k = 1; k < share; ++k)
179	48816x	t = t->next;
180	122x	b.head = t->next;
181	122x	b.count -= share;
182	122x	tails[i] = t;
183	122x	counts[i] = share;
184		}
185	1674x	b.low = b.count;
186		}
187
188	279x	node* spill = nullptr;
189		{
190	279x	std::lock_guard<std::mutex> lock(global_mutex());
191	279x	auto& gp = global();
192	1953x	for(std::size_t i = 0; i < num_classes; ++i)
193		{
194	1674x	if(counts[i] == 0)
195	1552x	continue;
196	122x	auto& g = gp.buckets[i];
197		// A share that doesn't fit whole is released whole: topping
198		// off a bucket already at least half full would mean walking
199		// the chain under the lock.
200	122x	if(g.count + counts[i] <= bucket_capacity(i))
201		{
202	43x	tails[i]->next = g.head;
203	43x	g.head = heads[i];
204	43x	g.count += counts[i];
205		}
206		else
207		{
208	79x	tails[i]->next = spill;
209	79x	spill = heads[i];
210		}
211		}
212	279x	}
213	39940x	while(spill != nullptr)
214		{
215	39661x	node* n = spill;
216	39661x	spill = n->next;
217	39661x	::operator delete(n);
218		}
219	279x	}
220
221		void*
222	102458x	recycling_memory_resource::do_allocate(std::size_t bytes, std::size_t alignment)
223		{
224	102458x	return allocate_fast(bytes, alignment);
225		}
226
227		void
228	102458x	recycling_memory_resource::do_deallocate(void* p, std::size_t bytes, std::size_t alignment)
229		{
230	102458x	deallocate_fast(p, bytes, alignment);
231	102458x	}
232
233		std::pmr::memory_resource*
234	4524x	get_recycling_memory_resource() noexcept
235		{
236	4524x	static recycling_memory_resource instance;
237	4524x	return &instance;
238		}
239
240		} // namespace capy
241		} // namespace boost
242