Halide 16.0.0
Halide compiler and libraries
region_allocator.h
Go to the documentation of this file.
1#ifndef HALIDE_RUNTIME_REGION_ALLOCATOR_H
2#define HALIDE_RUNTIME_REGION_ALLOCATOR_H
3
4#include "../HalideRuntime.h"
5#include "../printer.h"
6#include "memory_arena.h"
7#include "memory_resources.h"
8
9namespace Halide {
10namespace Runtime {
11namespace Internal {
12
13// --
14
15/** Allocator class interface for sub-allocating a contiguous
16 * memory block into smaller regions of memory. This class only
17 * manages the address creation for the regions -- allocation
18 * callback functions are used to request the memory from the
19 * necessary system or API calls. This class is intended to be
20 * used inside of a higher level memory management class that
21 * provides thread safety, policy management and API
22 * integration for a specific runtime API (eg Vulkan, OpenCL, etc)
23 */
25public:
26 // disable copy constructors and assignment
29
30 // disable non-factory based construction
31 RegionAllocator() = delete;
32 ~RegionAllocator() = delete;
33
34 // Allocators for the different types of memory we need to allocate
38 };
39
40 // Factory methods for creation / destruction
41 static RegionAllocator *create(void *user_context, BlockResource *block, const MemoryAllocators &ma);
42 static int destroy(void *user_context, RegionAllocator *region_allocator);
43
44 // Returns the allocator class instance for the given allocation (or nullptr)
45 static RegionAllocator *find_allocator(void *user_context, MemoryRegion *memory_region);
46
47 // Public interface methods
48 MemoryRegion *reserve(void *user_context, const MemoryRequest &request);
49 int release(void *user_context, MemoryRegion *memory_region); //< unmark and cache the region for reuse
50 int reclaim(void *user_context, MemoryRegion *memory_region); //< free the region and consolidate
51 int retain(void *user_context, MemoryRegion *memory_region); //< retain the region and increase usage count
52 bool collect(void *user_context); //< returns true if any blocks were removed
53 int release(void *user_context);
54 int destroy(void *user_context);
55
56 // Returns the currently managed block resource
58
59private:
60 // Initializes a new instance
61 int initialize(void *user_context, BlockResource *block, const MemoryAllocators &ma);
62
63 // Search through allocated block regions (Best-Fit)
64 BlockRegion *find_block_region(void *user_context, const MemoryRequest &request);
65
66 // Returns true if block region is unused and available
67 bool is_available(const BlockRegion *region) const;
68
69 // Returns true if neighbouring block regions to the given region can be coalesced into one
70 bool can_coalesce(const BlockRegion *region) const;
71
72 // Merges available neighbouring block regions into the given region
73 BlockRegion *coalesce_block_regions(void *user_context, BlockRegion *region);
74
75 // Returns true if the given region can be split to accomodate the given size
76 bool can_split(const BlockRegion *region, size_t size) const;
77
78 // Splits the given block region into a smaller region to accomodate the given size, followed by empty space for the remaining
79 BlockRegion *split_block_region(void *user_context, BlockRegion *region, size_t size, size_t alignment);
80
81 // Creates a new block region and adds it to the region list
82 BlockRegion *create_block_region(void *user_context, const MemoryProperties &properties, size_t offset, size_t size, bool dedicated);
83
84 // Creates a new block region and adds it to the region list
85 int destroy_block_region(void *user_context, BlockRegion *region);
86
87 // Invokes the allocation callback to allocate memory for the block region
88 int alloc_block_region(void *user_context, BlockRegion *region);
89
90 // Releases a block region and leaves it in the list for further allocations
91 int release_block_region(void *user_context, BlockRegion *region);
92
93 // Invokes the deallocation callback to free memory for the block region
94 int free_block_region(void *user_context, BlockRegion *region);
95
96 // Returns true if the given block region is the last region in the list
97 bool is_last_block_region(void *user_context, const BlockRegion *region) const;
98
99 // Returns true if the given block region is compatible with the given properties
100 bool is_compatible_block_region(const BlockRegion *region, const MemoryProperties &properties) const;
101
102 // Returns true if the given block region is suitable for the requested allocation
103 bool is_block_region_suitable_for_request(void *user_context, const BlockRegion *region, const MemoryRequest &request) const;
104
105 // Returns the number of active regions for the block;
106 size_t region_count(void *user_context) const;
107
108 BlockResource *block = nullptr;
109 MemoryArena *arena = nullptr;
110 MemoryAllocators allocators;
111};
112
113RegionAllocator *RegionAllocator::create(void *user_context, BlockResource *block_resource, const MemoryAllocators &allocators) {
114 halide_abort_if_false(user_context, allocators.system.allocate != nullptr);
115 RegionAllocator *result = reinterpret_cast<RegionAllocator *>(
116 allocators.system.allocate(user_context, sizeof(RegionAllocator)));
117
118 if (result == nullptr) {
119 return nullptr;
120 }
121
122 result->initialize(user_context, block_resource, allocators);
123 return result;
124}
125
126int RegionAllocator::destroy(void *user_context, RegionAllocator *instance) {
127 halide_abort_if_false(user_context, instance != nullptr);
128 const MemoryAllocators &allocators = instance->allocators;
129 instance->destroy(user_context);
130 halide_abort_if_false(user_context, allocators.system.deallocate != nullptr);
131 allocators.system.deallocate(user_context, instance);
132 return 0;
133}
134
135int RegionAllocator::initialize(void *user_context, BlockResource *mb, const MemoryAllocators &ma) {
136 block = mb;
137 allocators = ma;
138 arena = MemoryArena::create(user_context, {sizeof(BlockRegion), MemoryArena::default_capacity, 0}, allocators.system);
139 halide_abort_if_false(user_context, arena != nullptr);
140 block->allocator = this;
141 block->regions = create_block_region(
142 user_context,
143 block->memory.properties,
144 0, block->memory.size,
145 block->memory.dedicated);
146 return 0;
147}
148
149MemoryRegion *RegionAllocator::reserve(void *user_context, const MemoryRequest &request) {
150 halide_abort_if_false(user_context, request.size > 0);
151 size_t actual_alignment = conform_alignment(request.alignment, block->memory.properties.alignment);
152 size_t actual_size = conform_size(request.offset, request.size, actual_alignment, block->memory.properties.nearest_multiple);
153 size_t remaining = block->memory.size - block->reserved;
154 if (remaining < actual_size) {
155#ifdef DEBUG_RUNTIME_INTERNAL
156 debug(user_context) << "RegionAllocator: Unable to reserve more memory from block "
157 << "-- requested size (" << (int32_t)(request.size) << " bytes) "
158 << "greater than available (" << (int32_t)(remaining) << " bytes)!\n";
159#endif
160 return nullptr;
161 }
162
163 BlockRegion *block_region = find_block_region(user_context, request);
164 if (block_region == nullptr) {
165#ifdef DEBUG_RUNTIME_INTERNAL
166 debug(user_context) << "RegionAllocator: Failed to locate region for requested size ("
167 << (int32_t)(request.size) << " bytes)!\n";
168#endif
169 return nullptr;
170 }
171
172 if (can_split(block_region, request.size)) {
173#ifdef DEBUG_RUNTIME_INTERNAL
174 debug(user_context) << "RegionAllocator: Splitting region of size ( " << (int32_t)(block_region->memory.size) << ") "
175 << "to accomodate requested size (" << (int32_t)(request.size) << " bytes)!\n";
176#endif
177 split_block_region(user_context, block_region, request.size, request.alignment);
178 }
179
180 alloc_block_region(user_context, block_region);
181 return reinterpret_cast<MemoryRegion *>(block_region);
182}
183
184int RegionAllocator::release(void *user_context, MemoryRegion *memory_region) {
185 BlockRegion *block_region = reinterpret_cast<BlockRegion *>(memory_region);
186 halide_abort_if_false(user_context, block_region != nullptr);
187 halide_abort_if_false(user_context, block_region->block_ptr == block);
188 if (block_region->usage_count > 0) {
189 block_region->usage_count--;
190 }
191 return release_block_region(user_context, block_region);
192}
193
194int RegionAllocator::reclaim(void *user_context, MemoryRegion *memory_region) {
195 BlockRegion *block_region = reinterpret_cast<BlockRegion *>(memory_region);
196 halide_abort_if_false(user_context, block_region != nullptr);
197 halide_abort_if_false(user_context, block_region->block_ptr == block);
198 if (block_region->usage_count > 0) {
199 block_region->usage_count--;
200 }
201 release_block_region(user_context, block_region);
202 free_block_region(user_context, block_region);
203 if (can_coalesce(block_region)) {
204 block_region = coalesce_block_regions(user_context, block_region);
205 }
206 return 0;
207}
208
209int RegionAllocator::retain(void *user_context, MemoryRegion *memory_region) {
210 BlockRegion *block_region = reinterpret_cast<BlockRegion *>(memory_region);
211 halide_abort_if_false(user_context, block_region != nullptr);
212 halide_abort_if_false(user_context, block_region->block_ptr == block);
213 block_region->usage_count++;
214 return 0;
215}
216
218 BlockRegion *block_region = reinterpret_cast<BlockRegion *>(memory_region);
219 if (block_region == nullptr) {
220 return nullptr;
221 }
222 if (block_region->block_ptr == nullptr) {
223 return nullptr;
224 }
225 return block_region->block_ptr->allocator;
226}
227
228bool RegionAllocator::is_last_block_region(void *user_context, const BlockRegion *region) const {
229 return ((region == nullptr) || (region == region->next_ptr) || (region->next_ptr == nullptr));
230}
231
232bool RegionAllocator::is_block_region_suitable_for_request(void *user_context, const BlockRegion *region, const MemoryRequest &request) const {
233 if (!is_available(region)) {
234#ifdef DEBUG_RUNTIME_INTERNAL
235 debug(user_context) << "RegionAllocator: skipping block region ... not available! "
236 << " block_region=" << (void *)region << "\n";
237#endif
238 return false;
239 }
240
241 // skip incompatible block regions for this request
242 if (!is_compatible_block_region(region, request.properties)) {
243#ifdef DEBUG_RUNTIME_INTERNAL
244 debug(user_context) << "RegionAllocator: skipping block region ... incompatible properties! "
245 << " block_region=" << (void *)region << "\n";
246#endif
247 return false;
248 }
249
250 size_t actual_alignment = conform_alignment(request.alignment, block->memory.properties.alignment);
251 size_t actual_size = conform_size(region->memory.offset, request.size, actual_alignment, block->memory.properties.nearest_multiple);
252
253 // is the adjusted size larger than the current region?
254 if (actual_size > region->memory.size) {
255#ifdef DEBUG_RUNTIME_INTERNAL
256 debug(user_context) << "RegionAllocator: skipping block region ... not enough space for adjusted size! "
257 << " block_region=" << (void *)region << "\n";
258#endif
259 return false;
260 }
261
262 // will the adjusted size fit within the remaining unallocated space?
263 if ((actual_size + block->reserved) <= block->memory.size) {
264#ifdef DEBUG_RUNTIME_INTERNAL
265 debug(user_context) << "RegionAllocator: found suitable block region! "
266 << " block_region=" << (void *)region << "\n";
267#endif
268 return true; // you betcha
269 }
270
271 return false;
272}
273
274BlockRegion *RegionAllocator::find_block_region(void *user_context, const MemoryRequest &request) {
275 BlockRegion *block_region = block->regions;
276 while (block_region != nullptr) {
277 if (is_block_region_suitable_for_request(user_context, block_region, request)) {
278#ifdef DEBUG_RUNTIME_INTERNAL
279 debug(user_context) << "RegionAllocator: found suitable region ...\n"
280 << " user_context=" << (void *)(user_context) << "\n"
281 << " block_resource=" << (void *)block << "\n"
282 << " block_size=" << (uint32_t)block->memory.size << "\n"
283 << " block_reserved=" << (uint32_t)block->reserved << "\n"
284 << " requested_size=" << (uint32_t)request.size << "\n"
285 << " requested_is_dedicated=" << (request.dedicated ? "true" : "false") << "\n"
286 << " requested_usage=" << halide_memory_usage_name(request.properties.usage) << "\n"
287 << " requested_caching=" << halide_memory_caching_name(request.properties.caching) << "\n"
288 << " requested_visibility=" << halide_memory_visibility_name(request.properties.visibility) << "\n";
289#endif
290 return block_region;
291 }
292
293 if (is_last_block_region(user_context, block_region)) {
294 block_region = nullptr; // end of list ... nothing found
295 break;
296 }
297 block_region = block_region->next_ptr;
298 }
299
300 if (block_region == nullptr) {
301#ifdef DEBUG_RUNTIME_INTERNAL
302 debug(user_context) << "RegionAllocator: couldn't find suitable region!\n"
303 << " user_context=" << (void *)(user_context) << "\n"
304 << " requested_size=" << (uint32_t)request.size << "\n"
305 << " requested_is_dedicated=" << (request.dedicated ? "true" : "false") << "\n"
306 << " requested_usage=" << halide_memory_usage_name(request.properties.usage) << "\n"
307 << " requested_caching=" << halide_memory_caching_name(request.properties.caching) << "\n"
308 << " requested_visibility=" << halide_memory_visibility_name(request.properties.visibility) << "\n";
309#endif
310 }
311
312 return block_region;
313}
314
315bool RegionAllocator::is_available(const BlockRegion *block_region) const {
316 if (block_region == nullptr) {
317 return false;
318 }
319 if (block_region->usage_count > 0) {
320 return false;
321 }
322 if (block_region->status != AllocationStatus::Available) {
323 return false;
324 }
325 return true;
326}
327
328bool RegionAllocator::can_coalesce(const BlockRegion *block_region) const {
329 if (!is_available(block_region)) {
330 return false;
331 }
332 if (is_available(block_region->prev_ptr)) {
333 return true;
334 }
335 if (is_available(block_region->next_ptr)) {
336 return true;
337 }
338 return false;
339}
340
341BlockRegion *RegionAllocator::coalesce_block_regions(void *user_context, BlockRegion *block_region) {
342
343 if ((block_region->usage_count == 0) && (block_region->memory.handle != nullptr)) {
344#ifdef DEBUG_RUNTIME_INTERNAL
345 debug(user_context) << "Freeing region ("
346 << "block_ptr=" << (void *)block_region->block_ptr << " "
347 << "block_region=" << (void *)block_region << " "
348 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
349 << "block_reserved=" << (uint32_t)block->reserved << " "
350 << ")\n";
351#endif
352 halide_abort_if_false(user_context, allocators.region.deallocate != nullptr);
353 MemoryRegion *memory_region = &(block_region->memory);
354 allocators.region.deallocate(user_context, memory_region);
355 block_region->memory.handle = nullptr;
356 }
357
358 BlockRegion *prev_region = block_region->prev_ptr;
359 if (is_available(prev_region) && (prev_region != block_region)) {
360
361#ifdef DEBUG_RUNTIME_INTERNAL
362 debug(user_context) << "RegionAllocator: Coalescing "
363 << "previous region (offset=" << (int32_t)prev_region->memory.offset << " size=" << (int32_t)(prev_region->memory.size) << " bytes) "
364 << "into current region (offset=" << (int32_t)block_region->memory.offset << " size=" << (int32_t)(block_region->memory.size) << " bytes)\n!";
365#endif
366
367 prev_region->next_ptr = block_region->next_ptr;
368 if (block_region->next_ptr) {
369 block_region->next_ptr->prev_ptr = prev_region;
370 }
371 prev_region->memory.size += block_region->memory.size;
372 destroy_block_region(user_context, block_region);
373 block_region = prev_region;
374 }
375
376 BlockRegion *next_region = block_region->next_ptr;
377 if (is_available(next_region) && (next_region != block_region)) {
378
379#ifdef DEBUG_RUNTIME_INTERNAL
380 debug(user_context) << "RegionAllocator: Coalescing "
381 << "next region (offset=" << (int32_t)next_region->memory.offset << " size=" << (int32_t)(next_region->memory.size) << " bytes) "
382 << "into current region (offset=" << (int32_t)block_region->memory.offset << " size=" << (int32_t)(block_region->memory.size) << " bytes)!\n";
383#endif
384
385 if (next_region->next_ptr) {
386 next_region->next_ptr->prev_ptr = block_region;
387 }
388 block_region->next_ptr = next_region->next_ptr;
389 block_region->memory.size += next_region->memory.size;
390 destroy_block_region(user_context, next_region);
391 }
392
393 return block_region;
394}
395
396bool RegionAllocator::can_split(const BlockRegion *block_region, size_t size) const {
397 return (block_region && (block_region->memory.size > size) && (block_region->usage_count == 0));
398}
399
400BlockRegion *RegionAllocator::split_block_region(void *user_context, BlockRegion *block_region, size_t size, size_t alignment) {
401
402 if ((block_region->usage_count == 0) && (block_region->memory.handle != nullptr)) {
403#ifdef DEBUG_RUNTIME_INTERNAL
404 debug(user_context) << "RegionAllocator: Split deallocate region ("
405 << "block_ptr=" << (void *)block_region->block_ptr << " "
406 << "block_region=" << (void *)block_region << " "
407 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
408 << "block_reserved=" << (uint32_t)block_region->block_ptr->reserved << " "
409 << ")\n";
410#endif
411 halide_abort_if_false(user_context, allocators.region.deallocate != nullptr);
412 MemoryRegion *memory_region = &(block_region->memory);
413 allocators.region.deallocate(user_context, memory_region);
414 block_region->memory.handle = nullptr;
415 }
416
417 size_t actual_alignment = conform_alignment(alignment, block->memory.properties.alignment);
418 size_t split_size = conform_size(block_region->memory.offset, size, actual_alignment, block->memory.properties.nearest_multiple);
419 size_t split_offset = aligned_offset(block_region->memory.offset + size, actual_alignment);
420 size_t empty_size = block_region->memory.size - split_size;
421
422#ifdef DEBUG_RUNTIME_INTERNAL
423 debug(user_context) << "RegionAllocator: Conforming size and alignment \n"
424 << " requested_size=" << (uint32_t)size << "\n"
425 << " split_size=" << (uint32_t)split_size << "\n"
426 << " requested_alignment=" << (uint32_t)alignment << " "
427 << " required_alignment=" << (uint32_t)block->memory.properties.alignment << " "
428 << " actual_alignment=" << (uint32_t)actual_alignment << ")\n";
429#endif
430
431#ifdef DEBUG_RUNTIME_INTERNAL
432 debug(user_context) << "RegionAllocator: Splitting "
433 << "current region (offset=" << (int32_t)block_region->memory.offset << " size=" << (int32_t)(block_region->memory.size) << " bytes) "
434 << "to create empty region (offset=" << (int32_t)split_offset << " size=" << (int32_t)(empty_size) << " bytes)!\n";
435#endif
436
437 BlockRegion *next_region = block_region->next_ptr;
438 BlockRegion *empty_region = create_block_region(user_context,
439 block_region->memory.properties,
440 split_offset, empty_size,
441 block_region->memory.dedicated);
442 halide_abort_if_false(user_context, empty_region != nullptr);
443
444 empty_region->next_ptr = next_region;
445 if (next_region) {
446 next_region->prev_ptr = empty_region;
447 }
448 empty_region->prev_ptr = block_region;
449 block_region->next_ptr = empty_region;
450 block_region->memory.size -= empty_size;
451 return empty_region;
452}
453
454BlockRegion *RegionAllocator::create_block_region(void *user_context, const MemoryProperties &properties, size_t offset, size_t size, bool dedicated) {
455#ifdef DEBUG_RUNTIME_INTERNAL
456 debug(user_context) << "RegionAllocator: Creating block region ("
457 << "user_context=" << (void *)(user_context) << " "
458 << "offset=" << (uint32_t)offset << " "
459 << "size=" << (uint32_t)size << " "
460 << "alignment=" << (uint32_t)properties.alignment << " "
461 << "dedicated=" << (dedicated ? "true" : "false") << " "
462 << "usage=" << halide_memory_usage_name(properties.usage) << " "
463 << "caching=" << halide_memory_caching_name(properties.caching) << " "
464 << "visibility=" << halide_memory_visibility_name(properties.visibility) << ") ...\n";
465#endif
466
467 BlockRegion *block_region = static_cast<BlockRegion *>(arena->reserve(user_context, true));
468 if (block_region == nullptr) {
469 error(user_context) << "RegionAllocator: Failed to allocate new block region!\n";
470 return nullptr;
471 }
472
473#ifdef DEBUG_RUNTIME_INTERNAL
474 debug(user_context) << "RegionAllocator: Added block region ("
475 << "user_context=" << (void *)(user_context) << " "
476 << "block_region=" << (void *)(block_region) << ") ...\n";
477#endif
478
479 size_t actual_alignment = conform_alignment(properties.alignment, block->memory.properties.alignment);
480 size_t actual_size = conform_size(offset, size, actual_alignment, block->memory.properties.nearest_multiple);
481 size_t actual_offset = aligned_offset(offset, actual_alignment);
482
483 block_region->memory.handle = nullptr;
484 block_region->memory.offset = actual_offset;
485 block_region->memory.size = actual_size;
486 block_region->memory.properties = properties;
487 block_region->memory.dedicated = dedicated;
488 block_region->status = AllocationStatus::Available;
489 block_region->block_ptr = block;
490 block_region->usage_count = 0;
491
492#ifdef DEBUG_RUNTIME_INTERNAL
493 debug(user_context) << "Creating region ("
494 << "block_ptr=" << (void *)block_region->block_ptr << " "
495 << "block_region=" << (void *)block_region << " "
496 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
497 << ")\n";
498#endif
499
500 return block_region;
501}
502
503int RegionAllocator::release_block_region(void *user_context, BlockRegion *block_region) {
504#ifdef DEBUG_RUNTIME_INTERNAL
505 debug(user_context) << "RegionAllocator: Releasing block region ("
506 << "user_context=" << (void *)(user_context) << " "
507 << "block_region=" << (void *)(block_region) << ") ...\n";
508#endif
509 if (block_region == nullptr) {
510 return 0;
511 }
512
513 if (block_region->usage_count > 0) {
514 return 0;
515 }
516
517 if (block_region->status != AllocationStatus::Available) {
518
519#ifdef DEBUG_RUNTIME_INTERNAL
520 debug(user_context) << "Releasing region ("
521 << "block_ptr=" << (void *)block_region->block_ptr << " "
522 << "block_region=" << (void *)block_region << " "
523 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
524 << "block_reserved=" << (uint32_t)(block->reserved - block_region->memory.size) << " "
525 << ")\n";
526#endif
527
528 block->reserved -= block_region->memory.size;
529 }
530 block_region->status = AllocationStatus::Available;
531 return 0;
532}
533
534int RegionAllocator::destroy_block_region(void *user_context, BlockRegion *block_region) {
535#ifdef DEBUG_RUNTIME_INTERNAL
536 debug(user_context) << "RegionAllocator: Destroying block region ("
537 << "user_context=" << (void *)(user_context) << " "
538 << "block_region=" << (void *)(block_region) << ") ...\n";
539#endif
540
541 block_region->usage_count = 0;
542 release_block_region(user_context, block_region);
543 free_block_region(user_context, block_region);
544 arena->reclaim(user_context, block_region);
545 return 0;
546}
547
548int RegionAllocator::alloc_block_region(void *user_context, BlockRegion *block_region) {
549#ifdef DEBUG_RUNTIME_INTERNAL
550 debug(user_context) << "RegionAllocator: Allocating region (user_context=" << (void *)(user_context)
551 << " size=" << (int32_t)(block_region->memory.size)
552 << " offset=" << (int32_t)block_region->memory.offset << ")!\n";
553#endif
554 halide_abort_if_false(user_context, allocators.region.allocate != nullptr);
555 halide_abort_if_false(user_context, block_region->status == AllocationStatus::Available);
556 int error_code = 0;
557 MemoryRegion *memory_region = &(block_region->memory);
558 if (memory_region->handle == nullptr) {
559 error_code = allocators.region.allocate(user_context, memory_region);
560 memory_region->is_owner = true;
561
562#ifdef DEBUG_RUNTIME_INTERNAL
563 debug(user_context) << "Allocating region ("
564 << "block_ptr=" << (void *)block_region->block_ptr << " "
565 << "block_region=" << (void *)block_region << " "
566 << "memory_offset=" << (uint32_t)(block_region->memory.offset) << " "
567 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
568 << "block_reserved=" << (uint32_t)block->reserved << " "
569 << ")\n";
570#endif
571
572 } else {
573
574#ifdef DEBUG_RUNTIME_INTERNAL
575 debug(user_context) << "Re-using region ("
576 << "block_ptr=" << (void *)block_region->block_ptr << " "
577 << "block_region=" << (void *)block_region << " "
578 << "memory_offset=" << (uint32_t)(block_region->memory.offset) << " "
579 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
580 << "block_reserved=" << (uint32_t)block->reserved << " "
581 << ")\n";
582#endif
583 }
584 block_region->status = block_region->memory.dedicated ? AllocationStatus::Dedicated : AllocationStatus::InUse;
585 block->reserved += block_region->memory.size;
586 return error_code;
587}
588
589int RegionAllocator::free_block_region(void *user_context, BlockRegion *block_region) {
590#ifdef DEBUG_RUNTIME_INTERNAL
591 debug(user_context) << "RegionAllocator: Freeing block region ("
592 << "user_context=" << (void *)(user_context) << " "
593 << "block_region=" << (void *)(block_region) << " "
594 << "status=" << (uint32_t)block_region->status << " "
595 << "usage_count=" << (uint32_t)block_region->usage_count << ") ...\n";
596#endif
597 if ((block_region->usage_count == 0) && (block_region->memory.handle != nullptr)) {
598#ifdef DEBUG_RUNTIME_INTERNAL
599 debug(user_context) << "Freeing region ("
600 << "block_ptr=" << (void *)block_region->block_ptr << " "
601 << "block_region=" << (void *)block_region << " "
602 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
603 << "block_reserved=" << (uint32_t)block->reserved << " "
604 << ")\n";
605#endif
606 halide_abort_if_false(user_context, allocators.region.deallocate != nullptr);
607 MemoryRegion *memory_region = &(block_region->memory);
608 allocators.region.deallocate(user_context, memory_region);
609 block_region->memory.size = 0;
610 block_region->memory.offset = 0;
611 block_region->memory.handle = nullptr;
612 }
613 block_region->usage_count = 0;
614 block_region->status = AllocationStatus::Available;
615 return 0;
616}
617
618int RegionAllocator::release(void *user_context) {
619#ifdef DEBUG_RUNTIME_INTERNAL
620 debug(user_context) << "RegionAllocator: Releasing all regions ("
621 << "user_context=" << (void *)(user_context) << ") ...\n";
622#endif
623
624 BlockRegion *block_region = block->regions;
625 while (block_region != nullptr) {
626 release_block_region(user_context, block_region);
627 if (is_last_block_region(user_context, block_region)) {
628 break;
629 }
630 block_region = block_region->next_ptr;
631 }
632 return 0;
633}
634
635bool RegionAllocator::collect(void *user_context) {
636#ifdef DEBUG_RUNTIME_INTERNAL
637 debug(user_context) << "RegionAllocator: Collecting free block regions ("
638 << "user_context=" << (void *)(user_context) << ") ...\n";
639
640 uint32_t count = 0;
641 uint64_t reserved = block->reserved;
642 debug(user_context) << " collecting unused regions ("
643 << "block_ptr=" << (void *)block << " "
644 << "block_reserved=" << (uint32_t)block->reserved << " "
645 << ")\n";
646#endif
647
648 bool has_collected = false;
649 BlockRegion *block_region = block->regions;
650 while (block_region != nullptr) {
651 if (can_coalesce(block_region)) {
652#ifdef DEBUG_RUNTIME_INTERNAL
653 count++;
654 debug(user_context) << " collecting region ("
655 << "block_ptr=" << (void *)block_region->block_ptr << " "
656 << "block_region=" << (void *)block_region << " "
657 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
658 << "block_reserved=" << (uint32_t)block->reserved << " "
659 << ")\n";
660#endif
661 block_region = coalesce_block_regions(user_context, block_region);
662 has_collected = true;
663 }
664 if (is_last_block_region(user_context, block_region)) {
665 break;
666 }
667 block_region = block_region->next_ptr;
668 }
669
670 if (has_collected) {
671#ifdef DEBUG_RUNTIME_INTERNAL
672 debug(user_context) << " collected unused regions ("
673 << "block_ptr=" << (void *)block << " "
674 << "region_count=" << (uint32_t)count << " "
675 << "collected=" << (uint32_t)(reserved - block->reserved) << " "
676 << ")\n";
677#endif
678 }
679 return has_collected;
680}
681
682int RegionAllocator::destroy(void *user_context) {
683#ifdef DEBUG_RUNTIME_INTERNAL
684 debug(user_context) << "RegionAllocator: Destroying all block regions ("
685 << "user_context=" << (void *)(user_context) << ") ...\n";
686#endif
687 for (BlockRegion *block_region = block->regions; block_region != nullptr;) {
688
689 if (is_last_block_region(user_context, block_region)) {
690 destroy_block_region(user_context, block_region);
691 block_region = nullptr;
692 } else {
693 BlockRegion *prev_region = block_region;
694 block_region = block_region->next_ptr;
695 destroy_block_region(user_context, prev_region);
696 }
697 }
698 block->reserved = 0;
699 block->regions = nullptr;
700 block->allocator = nullptr;
701 MemoryArena::destroy(user_context, arena);
702 arena = nullptr;
703 return 0;
704}
705
706bool RegionAllocator::is_compatible_block_region(const BlockRegion *block_region, const MemoryProperties &properties) const {
707 if (properties.caching != MemoryCaching::DefaultCaching) {
708 if (properties.caching != block_region->memory.properties.caching) {
709 return false;
710 }
711 }
712
714 if (properties.visibility != block_region->memory.properties.visibility) {
715 return false;
716 }
717 }
718
719 if (properties.usage != MemoryUsage::DefaultUsage) {
720 if (properties.usage != block_region->memory.properties.usage) {
721 return false;
722 }
723 }
724
725 return true;
726}
727
728size_t RegionAllocator::region_count(void *user_context) const {
729 if (block == nullptr) {
730 return 0;
731 }
732 size_t count = 0;
733 for (BlockRegion const *region = block->regions; !is_last_block_region(user_context, region); region = region->next_ptr) {
734 ++count;
735 }
736 return count;
737}
738
740 return block;
741}
742
743// --
744
745} // namespace Internal
746} // namespace Runtime
747} // namespace Halide
748
749#endif // HALIDE_RUNTIME_REGION_ALLOCATOR_H
void * reserve(void *user_context, bool initialize=false)
Definition: memory_arena.h:155
static MemoryArena * create(void *user_context, const Config &config, const SystemMemoryAllocatorFns &allocator=default_allocator())
Definition: memory_arena.h:101
static constexpr uint32_t default_capacity
Definition: memory_arena.h:24
static void destroy(void *user_context, MemoryArena *arena)
Definition: memory_arena.h:115
void reclaim(void *user_context, void *ptr)
Definition: memory_arena.h:182
Allocator class interface for sub-allocating a contiguous memory block into smaller regions of memory...
MemoryRegion * reserve(void *user_context, const MemoryRequest &request)
RegionAllocator & operator=(const RegionAllocator &)=delete
RegionAllocator(const RegionAllocator &)=delete
static int destroy(void *user_context, RegionAllocator *region_allocator)
static RegionAllocator * find_allocator(void *user_context, MemoryRegion *memory_region)
int retain(void *user_context, MemoryRegion *memory_region)
int reclaim(void *user_context, MemoryRegion *memory_region)
int release(void *user_context, MemoryRegion *memory_region)
static RegionAllocator * create(void *user_context, BlockResource *block, const MemoryAllocators &ma)
WEAK const char * halide_memory_caching_name(MemoryCaching value)
WEAK const char * halide_memory_usage_name(MemoryUsage value)
WEAK const char * halide_memory_visibility_name(MemoryVisibility value)
ALWAYS_INLINE size_t conform_alignment(size_t requested, size_t required)
ALWAYS_INLINE size_t conform_size(size_t offset, size_t size, size_t alignment, size_t nearest_multiple)
ALWAYS_INLINE size_t aligned_offset(size_t offset, size_t alignment)
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
@ Internal
Not visible externally, similar to 'static' linkage in C.
unsigned __INT64_TYPE__ uint64_t
signed __INT32_TYPE__ int32_t
unsigned __INT32_TYPE__ uint32_t
#define halide_abort_if_false(user_context, cond)