TLA Line data Source code
1 : //
2 : // Copyright (c) 2025 Vinnie Falco (vinnie.falco@gmail.com)
3 : //
4 : // Distributed under the Boost Software License, Version 1.0. (See accompanying
5 : // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6 : //
7 : // Official repository: https://github.com/cppalliance/capy
8 : //
9 :
10 : #include <boost/capy/ex/recycling_memory_resource.hpp>
11 :
12 : #include <new>
13 : #include <tuple>
14 :
15 : namespace boost {
16 : namespace capy {
17 :
18 : // Instance destruction does nothing: the resource is stateless (all pools
19 : // are static). Global-pool cleanup is owned by global()'s holder, exactly
20 : // as in the original where the global pool's own destructor drained it,
21 : // independent of any instance lifetime.
22 HIT 327 : recycling_memory_resource::~recycling_memory_resource() = default;
23 :
24 : #if (defined(BOOST_CAPY_DYN_LINK) || defined(BOOST_ALL_DYN_LINK)) && \
25 : !defined(BOOST_CAPY_STATIC_LINK)
26 :
27 : recycling_memory_resource::pool&
28 2590777 : recycling_memory_resource::local() noexcept
29 : {
30 : static thread_local pool p;
31 2590777 : return p;
32 : }
33 :
34 : bool&
35 1144341 : recycling_memory_resource::thread_armed() noexcept
36 : {
37 : static thread_local bool armed = false;
38 1144341 : return armed;
39 : }
40 :
41 : #endif
42 :
43 : namespace {
44 :
45 : // Set once this thread's pool was drained at exit. Another thread_local
46 : // destroyed later can still free frames; they must not be cached.
47 : thread_local bool pool_drained = false;
48 :
49 : } // namespace
50 :
51 : bool
52 302819 : recycling_memory_resource::arm_thread_cleanup() noexcept
53 : {
54 : struct janitor
55 : {
56 373 : ~janitor()
57 : {
58 : // Return this thread's cached blocks to the OS so nothing
59 : // leaks at thread exit. The pool has a trivial dtor, so its
60 : // thread-local storage is still valid here.
61 373 : auto& lp = local();
62 2611 : for(auto& b : lp.buckets)
63 52545 : while(b.count > 0)
64 50307 : ::operator delete(b.pop());
65 373 : pool_drained = true;
66 373 : thread_armed() = false;
67 373 : }
68 : };
69 302819 : if(pool_drained)
70 5 : return false;
71 302814 : static thread_local janitor j;
72 302814 : std::ignore = j;
73 302814 : return true;
74 : }
75 :
76 : recycling_memory_resource::pool&
77 569107 : recycling_memory_resource::global() noexcept
78 : {
79 : // Holder gives the global pool a destructor (the trivial-dtor pool type
80 : // itself must stay guard-free for local()). Runs unconditionally at
81 : // process exit, mirroring the original global pool destructor, and is
82 : // locked because a worker thread may still be in a slow path. This path
83 : // is cold (slow paths only), so the holder's guard costs nothing hot.
84 : struct holder
85 : {
86 : pool p;
87 :
88 33 : ~holder()
89 : {
90 33 : std::lock_guard<std::mutex> lock(global_mutex());
91 231 : for(auto& b : p.buckets)
92 5320 : while(b.count > 0)
93 5122 : ::operator delete(b.pop());
94 33 : }
95 : };
96 569107 : static holder h;
97 569107 : return h.p;
98 : }
99 :
100 : std::mutex&
101 569140 : recycling_memory_resource::global_mutex() noexcept
102 : {
103 : // Never destroyed: it is locked during process-exit teardown (in
104 : // global()'s holder destructor), after a function-local `static
105 : // std::mutex` could itself have been destroyed. Placement-new into
106 : // static storage owns no heap allocation, so there is nothing to leak.
107 : alignas(std::mutex) static unsigned char storage[sizeof(std::mutex)];
108 : static std::mutex* const mtx =
109 569140 : ::new(static_cast<void*>(storage)) std::mutex();
110 569140 : return *mtx;
111 : }
112 :
113 : void*
114 302444 : recycling_memory_resource::allocate_slow(
115 : std::size_t rounded, std::size_t idx)
116 : {
117 : // Once this thread's pool was drained at exit, take one block and
118 : // leave the pool empty: nothing would release a refill.
119 302444 : bool const cache = arm_thread_cleanup();
120 302444 : void* p = nullptr;
121 302444 : std::size_t refilled = 0;
122 : {
123 302444 : std::lock_guard<std::mutex> lock(global_mutex());
124 302444 : auto& b = global().buckets[idx];
125 302444 : if(cache)
126 : {
127 302443 : auto& lb = local().buckets[idx];
128 302443 : p = b.pop(lb);
129 302443 : refilled = lb.count;
130 : }
131 : else
132 : {
133 1 : p = b.pop();
134 : }
135 302444 : }
136 302444 : if(p == nullptr)
137 302415 : return ::operator new(rounded);
138 :
139 : // A thread that only allocates never frees into its pool, so its
140 : // refills are what drive its decay passes.
141 29 : auto& lp = local();
142 29 : lp.arrivals += refilled;
143 29 : if(lp.arrivals >= decay_period)
144 7 : decay(lp);
145 29 : return p;
146 : }
147 :
148 : void
149 266384 : recycling_memory_resource::deallocate_slow(
150 : void* p, std::size_t idx)
151 : {
152 : {
153 266384 : std::lock_guard<std::mutex> lock(global_mutex());
154 266384 : if(global().buckets[idx].push(p, bucket_capacity(idx)))
155 59059 : return;
156 266384 : }
157 207325 : ::operator delete(p);
158 : }
159 :
160 : void
161 279 : recycling_memory_resource::decay(pool& lp) noexcept
162 : {
163 279 : lp.arrivals = 0;
164 :
165 : // Unlink each bucket's share before taking the lock, so the walk
166 : // through the blocks happens outside it.
167 279 : node* heads[num_classes] = {};
168 279 : node* tails[num_classes] = {};
169 279 : std::size_t counts[num_classes] = {};
170 1953 : for(std::size_t i = 0; i < num_classes; ++i)
171 : {
172 1674 : auto& b = lp.buckets[i];
173 1674 : std::size_t const share = (b.low + 1) / 2;
174 1674 : if(share > 0)
175 : {
176 122 : heads[i] = b.head;
177 122 : node* t = b.head;
178 48938 : for(std::size_t k = 1; k < share; ++k)
179 48816 : t = t->next;
180 122 : b.head = t->next;
181 122 : b.count -= share;
182 122 : tails[i] = t;
183 122 : counts[i] = share;
184 : }
185 1674 : b.low = b.count;
186 : }
187 :
188 279 : node* spill = nullptr;
189 : {
190 279 : std::lock_guard<std::mutex> lock(global_mutex());
191 279 : auto& gp = global();
192 1953 : for(std::size_t i = 0; i < num_classes; ++i)
193 : {
194 1674 : if(counts[i] == 0)
195 1552 : continue;
196 122 : auto& g = gp.buckets[i];
197 : // A share that doesn't fit whole is released whole: topping
198 : // off a bucket already at least half full would mean walking
199 : // the chain under the lock.
200 122 : if(g.count + counts[i] <= bucket_capacity(i))
201 : {
202 43 : tails[i]->next = g.head;
203 43 : g.head = heads[i];
204 43 : g.count += counts[i];
205 : }
206 : else
207 : {
208 79 : tails[i]->next = spill;
209 79 : spill = heads[i];
210 : }
211 : }
212 279 : }
213 39940 : while(spill != nullptr)
214 : {
215 39661 : node* n = spill;
216 39661 : spill = n->next;
217 39661 : ::operator delete(n);
218 : }
219 279 : }
220 :
221 : void*
222 102458 : recycling_memory_resource::do_allocate(std::size_t bytes, std::size_t alignment)
223 : {
224 102458 : return allocate_fast(bytes, alignment);
225 : }
226 :
227 : void
228 102458 : recycling_memory_resource::do_deallocate(void* p, std::size_t bytes, std::size_t alignment)
229 : {
230 102458 : deallocate_fast(p, bytes, alignment);
231 102458 : }
232 :
233 : std::pmr::memory_resource*
234 4524 : get_recycling_memory_resource() noexcept
235 : {
236 4524 : static recycling_memory_resource instance;
237 4524 : return &instance;
238 : }
239 :
240 : } // namespace capy
241 : } // namespace boost
|
