. /src /ex /recycling_memory_resource.cpp
100.00% Lines (101/101) 100.00% Functions (14/14)
[ ] prev/next file n p prev/next uncovered line
TLA Baseline Branch
Line Hits Code Line Hits Code
1   // 1   //
2   // Copyright (c) 2025 Vinnie Falco (vinnie.falco@gmail.com) 2   // Copyright (c) 2025 Vinnie Falco (vinnie.falco@gmail.com)
3   // 3   //
4   // Distributed under the Boost Software License, Version 1.0. (See accompanying 4   // Distributed under the Boost Software License, Version 1.0. (See accompanying
5   // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) 5   // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6   // 6   //
7   // Official repository: https://github.com/cppalliance/capy 7   // Official repository: https://github.com/cppalliance/capy
8   // 8   //
9   9  
10   #include <boost/capy/ex/recycling_memory_resource.hpp> 10   #include <boost/capy/ex/recycling_memory_resource.hpp>
11   11  
12   #include <new> 12   #include <new>
13   #include <tuple> 13   #include <tuple>
14   14  
15   namespace boost { 15   namespace boost {
16   namespace capy { 16   namespace capy {
17   17  
18   // Instance destruction does nothing: the resource is stateless (all pools 18   // Instance destruction does nothing: the resource is stateless (all pools
19   // are static). Global-pool cleanup is owned by global()'s holder, exactly 19   // are static). Global-pool cleanup is owned by global()'s holder, exactly
20   // as in the original where the global pool's own destructor drained it, 20   // as in the original where the global pool's own destructor drained it,
21   // independent of any instance lifetime. 21   // independent of any instance lifetime.
HITCBC 22   307 recycling_memory_resource::~recycling_memory_resource() = default; 22   327 recycling_memory_resource::~recycling_memory_resource() = default;
23   23  
24   #if (defined(BOOST_CAPY_DYN_LINK) || defined(BOOST_ALL_DYN_LINK)) && \ 24   #if (defined(BOOST_CAPY_DYN_LINK) || defined(BOOST_ALL_DYN_LINK)) && \
25   !defined(BOOST_CAPY_STATIC_LINK) 25   !defined(BOOST_CAPY_STATIC_LINK)
26   26  
27   recycling_memory_resource::pool& 27   recycling_memory_resource::pool&
HITCBC 28   643994 recycling_memory_resource::local() noexcept 28   2590777 recycling_memory_resource::local() noexcept
29   { 29   {
30   static thread_local pool p; 30   static thread_local pool p;
HITCBC 31   643994 return p; 31   2590777 return p;
32   } 32   }
33   33  
34   bool& 34   bool&
HITCBC 35   216915 recycling_memory_resource::thread_armed() noexcept 35   1144341 recycling_memory_resource::thread_armed() noexcept
36   { 36   {
37   static thread_local bool armed = false; 37   static thread_local bool armed = false;
HITCBC 38   216915 return armed; 38   1144341 return armed;
39   } 39   }
40   40  
41   #endif 41   #endif
42   42  
43   namespace { 43   namespace {
44   44  
45   // Set once this thread's pool was drained at exit. Another thread_local 45   // Set once this thread's pool was drained at exit. Another thread_local
46   // destroyed later can still free frames; they must not be cached. 46   // destroyed later can still free frames; they must not be cached.
47   thread_local bool pool_drained = false; 47   thread_local bool pool_drained = false;
48   48  
49   } // namespace 49   } // namespace
50   50  
51   bool 51   bool
HITCBC 52   210878 recycling_memory_resource::arm_thread_cleanup() noexcept 52   302819 recycling_memory_resource::arm_thread_cleanup() noexcept
53   { 53   {
54   struct janitor 54   struct janitor
55   { 55   {
HITCBC 56   354 ~janitor() 56   373 ~janitor()
57   { 57   {
58   // Return this thread's cached blocks to the OS so nothing 58   // Return this thread's cached blocks to the OS so nothing
59   // leaks at thread exit. The pool has a trivial dtor, so its 59   // leaks at thread exit. The pool has a trivial dtor, so its
60   // thread-local storage is still valid here. 60   // thread-local storage is still valid here.
HITCBC 61   354 auto& lp = local(); 61   373 auto& lp = local();
HITCBC 62   2478 for(auto& b : lp.buckets) 62   2611 for(auto& b : lp.buckets)
HITCBC 63   6640 while(b.count > 0) 63   52545 while(b.count > 0)
HITCBC 64   4516 ::operator delete(b.pop()); 64   50307 ::operator delete(b.pop());
HITCBC 65   354 pool_drained = true; 65   373 pool_drained = true;
HITCBC 66   354 thread_armed() = false; 66   373 thread_armed() = false;
HITCBC 67   354 } 67   373 }
68   }; 68   };
HITCBC 69   210878 if(pool_drained) 69   302819 if(pool_drained)
HITCBC 70   5 return false; 70   5 return false;
HITCBC 71   210873 static thread_local janitor j; 71   302814 static thread_local janitor j;
HITCBC 72   210873 std::ignore = j; 72   302814 std::ignore = j;
HITCBC 73   210873 return true; 73   302814 return true;
74   } 74   }
75   75  
76   recycling_memory_resource::pool& 76   recycling_memory_resource::pool&
HITCBC 77   420132 recycling_memory_resource::global() noexcept 77   569107 recycling_memory_resource::global() noexcept
78   { 78   {
79   // Holder gives the global pool a destructor (the trivial-dtor pool type 79   // Holder gives the global pool a destructor (the trivial-dtor pool type
80   // itself must stay guard-free for local()). Runs unconditionally at 80   // itself must stay guard-free for local()). Runs unconditionally at
81   // process exit, mirroring the original global pool destructor, and is 81   // process exit, mirroring the original global pool destructor, and is
82   // locked because a worker thread may still be in a slow path. This path 82   // locked because a worker thread may still be in a slow path. This path
83   // is cold (slow paths only), so the holder's guard costs nothing hot. 83   // is cold (slow paths only), so the holder's guard costs nothing hot.
84   struct holder 84   struct holder
85   { 85   {
86   pool p; 86   pool p;
87   87  
HITCBC 88   33 ~holder() 88   33 ~holder()
89   { 89   {
HITCBC 90   33 std::lock_guard<std::mutex> lock(global_mutex()); 90   33 std::lock_guard<std::mutex> lock(global_mutex());
HITCBC 91   231 for(auto& b : p.buckets) 91   231 for(auto& b : p.buckets)
HITCBC 92   321 while(b.count > 0) 92   5320 while(b.count > 0)
HITCBC 93   123 ::operator delete(b.pop()); 93   5122 ::operator delete(b.pop());
HITCBC 94   33 } 94   33 }
95   }; 95   };
HITCBC 96   420132 static holder h; 96   569107 static holder h;
HITCBC 97   420132 return h.p; 97   569107 return h.p;
98   } 98   }
99   99  
100   std::mutex& 100   std::mutex&
HITCBC 101   420165 recycling_memory_resource::global_mutex() noexcept 101   569140 recycling_memory_resource::global_mutex() noexcept
102   { 102   {
103   // Never destroyed: it is locked during process-exit teardown (in 103   // Never destroyed: it is locked during process-exit teardown (in
104   // global()'s holder destructor), after a function-local `static 104   // global()'s holder destructor), after a function-local `static
105   // std::mutex` could itself have been destroyed. Placement-new into 105   // std::mutex` could itself have been destroyed. Placement-new into
106   // static storage owns no heap allocation, so there is nothing to leak. 106   // static storage owns no heap allocation, so there is nothing to leak.
107   alignas(std::mutex) static unsigned char storage[sizeof(std::mutex)]; 107   alignas(std::mutex) static unsigned char storage[sizeof(std::mutex)];
108   static std::mutex* const mtx = 108   static std::mutex* const mtx =
HITCBC 109   420165 ::new(static_cast<void*>(storage)) std::mutex(); 109   569140 ::new(static_cast<void*>(storage)) std::mutex();
HITCBC 110   420165 return *mtx; 110   569140 return *mtx;
111   } 111   }
112   112  
113   void* 113   void*
HITCBC 114   210523 recycling_memory_resource::allocate_slow( 114   302444 recycling_memory_resource::allocate_slow(
115   std::size_t rounded, std::size_t idx) 115   std::size_t rounded, std::size_t idx)
116   { 116   {
117   // Once this thread's pool was drained at exit, take one block and 117   // Once this thread's pool was drained at exit, take one block and
118   // leave the pool empty: nothing would release a refill. 118   // leave the pool empty: nothing would release a refill.
HITCBC 119   210523 bool const cache = arm_thread_cleanup(); 119   302444 bool const cache = arm_thread_cleanup();
HITGNC   120 + 302444 void* p = nullptr;
HITGNC   121 + 302444 std::size_t refilled = 0;
120   { 122   {
HITCBC 121   210523 std::lock_guard<std::mutex> lock(global_mutex()); 123   302444 std::lock_guard<std::mutex> lock(global_mutex());
HITCBC 122   210523 auto& b = global().buckets[idx]; 124   302444 auto& b = global().buckets[idx];
HITCBC 123 - 210523 if(auto* p = cache ? b.pop(local().buckets[idx]) : b.pop()) 125 + 302444 if(cache)
ECB 124 - 591 return p; 126 + {
HITGNC   127 + 302443 auto& lb = local().buckets[idx];
HITGNC   128 + 302443 p = b.pop(lb);
HITGNC   129 + 302443 refilled = lb.count;
  130 + }
  131 + else
  132 + {
HITGNC   133 + 1 p = b.pop();
  134 + }
HITCBC 125   210523 } 135   302444 }
HITCBC 126 - 209932 return ::operator new(rounded); 136 + 302444 if(p == nullptr)
HITGNC   137 + 302415 return ::operator new(rounded);
  138 +
  139 + // A thread that only allocates never frees into its pool, so its
  140 + // refills are what drive its decay passes.
HITGNC   141 + 29 auto& lp = local();
HITGNC   142 + 29 lp.arrivals += refilled;
HITGNC   143 + 29 if(lp.arrivals >= decay_period)
HITGNC   144 + 7 decay(lp);
HITGNC   145 + 29 return p;
127   } 146   }
128   147  
129   void 148   void
HITCBC 130   209609 recycling_memory_resource::deallocate_slow( 149   266384 recycling_memory_resource::deallocate_slow(
131   void* p, std::size_t idx) 150   void* p, std::size_t idx)
132   { 151   {
133   { 152   {
HITCBC 134   209609 std::lock_guard<std::mutex> lock(global_mutex()); 153   266384 std::lock_guard<std::mutex> lock(global_mutex());
HITCBC 135 - 209609 if(global().buckets[idx].push(p)) 154 + 266384 if(global().buckets[idx].push(p, bucket_capacity(idx)))
HITCBC 136   4316 return; 155   59059 return;
HITCBC 137   209609 } 156   266384 }
HITCBC 138   205293 ::operator delete(p); 157   207325 ::operator delete(p);
  158 + }
  159 +
  160 + void
HITGNC   161 + 279 recycling_memory_resource::decay(pool& lp) noexcept
  162 + {
HITGNC   163 + 279 lp.arrivals = 0;
  164 +
  165 + // Unlink each bucket's share before taking the lock, so the walk
  166 + // through the blocks happens outside it.
HITGNC   167 + 279 node* heads[num_classes] = {};
HITGNC   168 + 279 node* tails[num_classes] = {};
HITGNC   169 + 279 std::size_t counts[num_classes] = {};
HITGNC   170 + 1953 for(std::size_t i = 0; i < num_classes; ++i)
  171 + {
HITGNC   172 + 1674 auto& b = lp.buckets[i];
HITGNC   173 + 1674 std::size_t const share = (b.low + 1) / 2;
HITGNC   174 + 1674 if(share > 0)
  175 + {
HITGNC   176 + 122 heads[i] = b.head;
HITGNC   177 + 122 node* t = b.head;
HITGNC   178 + 48938 for(std::size_t k = 1; k < share; ++k)
HITGNC   179 + 48816 t = t->next;
HITGNC   180 + 122 b.head = t->next;
HITGNC   181 + 122 b.count -= share;
HITGNC   182 + 122 tails[i] = t;
HITGNC   183 + 122 counts[i] = share;
  184 + }
HITGNC   185 + 1674 b.low = b.count;
  186 + }
  187 +
HITGNC   188 + 279 node* spill = nullptr;
  189 + {
HITGNC   190 + 279 std::lock_guard<std::mutex> lock(global_mutex());
HITGNC   191 + 279 auto& gp = global();
HITGNC   192 + 1953 for(std::size_t i = 0; i < num_classes; ++i)
  193 + {
HITGNC   194 + 1674 if(counts[i] == 0)
HITGNC   195 + 1552 continue;
HITGNC   196 + 122 auto& g = gp.buckets[i];
  197 + // A share that doesn't fit whole is released whole: topping
  198 + // off a bucket already at least half full would mean walking
  199 + // the chain under the lock.
HITGNC   200 + 122 if(g.count + counts[i] <= bucket_capacity(i))
  201 + {
HITGNC   202 + 43 tails[i]->next = g.head;
HITGNC   203 + 43 g.head = heads[i];
HITGNC   204 + 43 g.count += counts[i];
  205 + }
  206 + else
  207 + {
HITGNC   208 + 79 tails[i]->next = spill;
HITGNC   209 + 79 spill = heads[i];
  210 + }
  211 + }
HITGNC   212 + 279 }
HITGNC   213 + 39940 while(spill != nullptr)
  214 + {
HITGNC   215 + 39661 node* n = spill;
HITGNC   216 + 39661 spill = n->next;
HITGNC   217 + 39661 ::operator delete(n);
  218 + }
HITGIC 139   } 219   279 }
140   220  
141   void* 221   void*
HITCBC 142   102458 recycling_memory_resource::do_allocate(std::size_t bytes, std::size_t alignment) 222   102458 recycling_memory_resource::do_allocate(std::size_t bytes, std::size_t alignment)
143   { 223   {
HITCBC 144   102458 return allocate_fast(bytes, alignment); 224   102458 return allocate_fast(bytes, alignment);
145   } 225   }
146   226  
147   void 227   void
HITCBC 148   102458 recycling_memory_resource::do_deallocate(void* p, std::size_t bytes, std::size_t alignment) 228   102458 recycling_memory_resource::do_deallocate(void* p, std::size_t bytes, std::size_t alignment)
149   { 229   {
HITCBC 150   102458 deallocate_fast(p, bytes, alignment); 230   102458 deallocate_fast(p, bytes, alignment);
HITCBC 151   102458 } 231   102458 }
152   232  
153   std::pmr::memory_resource* 233   std::pmr::memory_resource*
HITCBC 154   4525 get_recycling_memory_resource() noexcept 234   4524 get_recycling_memory_resource() noexcept
155   { 235   {
HITCBC 156   4525 static recycling_memory_resource instance; 236   4524 static recycling_memory_resource instance;
HITCBC 157   4525 return &instance; 237   4524 return &instance;
158   } 238   }
159   239  
160   } // namespace capy 240   } // namespace capy
161   } // namespace boost 241   } // namespace boost