Line data Source code
1 : /**
2 : Copyright (c) 2021-2022 Roman Katuntsev <sbkarr@stappler.org>
3 : Copyright (c) 2023 Stappler LLC <admin@stappler.dev>
4 :
5 : Permission is hereby granted, free of charge, to any person obtaining a copy
6 : of this software and associated documentation files (the "Software"), to deal
7 : in the Software without restriction, including without limitation the rights
8 : to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 : copies of the Software, and to permit persons to whom the Software is
10 : furnished to do so, subject to the following conditions:
11 :
12 : The above copyright notice and this permission notice shall be included in
13 : all copies or substantial portions of the Software.
14 :
15 : THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 : IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 : FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 : AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 : LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 : OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 : THE SOFTWARE.
22 : **/
23 :
24 : #include "XLVkAllocator.h"
25 : #include "XLVkInstance.h"
26 : #include "XLVkDevice.h"
27 : #include "XLVkObject.h"
28 :
29 : namespace STAPPLER_VERSIONIZED stappler::xenolith::vk {
30 :
31 3488116 : static uint32_t Allocator_getTypeScoreInternal(const Allocator::MemHeap &heap, const Allocator::MemType &type, AllocationUsage usage) {
32 3488116 : switch (usage) {
33 2815794 : case AllocationUsage::DeviceLocal:
34 : case AllocationUsage::DeviceLocalLazilyAllocated:
35 2815794 : switch (heap.type) {
36 0 : case Allocator::MemHeapType::DeviceLocal:
37 0 : if (type.isDeviceLocal()) {
38 0 : uint32_t ret = 32;
39 0 : if (type.isHostVisible()) { ret -= 2; }
40 0 : if (type.isHostCoherent()) { ret -= 3; }
41 0 : if (type.isHostCached()) { ret -= 4; }
42 0 : if (usage == AllocationUsage::DeviceLocalLazilyAllocated && type.isLazilyAllocated()) { ret += 12; }
43 0 : return ret;
44 : }
45 0 : return 0;
46 : break;
47 1126577 : case Allocator::MemHeapType::DeviceLocalHostVisible:
48 1126577 : if (type.isDeviceLocal()) {
49 1126577 : uint32_t ret = 24;
50 1126577 : if (type.isHostVisible()) { ret -= 2; }
51 1126577 : if (type.isHostCoherent()) { ret -= 3; }
52 1126577 : if (type.isHostCached()) { ret -= 4; }
53 1126577 : if (usage == AllocationUsage::DeviceLocalLazilyAllocated && type.isLazilyAllocated()) { ret += 12; }
54 1126577 : return ret;
55 : }
56 0 : return 0;
57 : break;
58 1689217 : case Allocator::MemHeapType::HostLocal:
59 1689217 : return 0;
60 : break;
61 : }
62 0 : break;
63 523647 : case AllocationUsage::DeviceLocalHostVisible:
64 523647 : switch (heap.type) {
65 209521 : case Allocator::MemHeapType::DeviceLocalHostVisible:
66 209521 : if (type.isDeviceLocal() && type.isHostVisible()) {
67 104766 : uint32_t ret = 32;
68 104766 : if (type.isHostCoherent()) { ret -= 3; }
69 104765 : if (type.isHostCached()) { ret -= 4; }
70 104765 : return ret;
71 : }
72 104761 : return 0;
73 : break;
74 0 : case Allocator::MemHeapType::DeviceLocal:
75 0 : return 0;
76 : break;
77 314234 : case Allocator::MemHeapType::HostLocal:
78 314234 : if (type.isHostVisible()) {
79 209485 : return 1;
80 : }
81 104743 : return 0;
82 : break;
83 : }
84 0 : break;
85 22585 : case AllocationUsage::HostTransitionSource:
86 22585 : switch (heap.type) {
87 13551 : case Allocator::MemHeapType::HostLocal:
88 13551 : if (type.isHostVisible()) {
89 9034 : uint32_t ret = 32;
90 9034 : if (type.isHostCoherent()) { ret += 3; }
91 9034 : if (type.isHostCached()) { ret -= 4; }
92 9034 : return ret;
93 : }
94 4517 : return 0;
95 : break;
96 9034 : case Allocator::MemHeapType::DeviceLocalHostVisible:
97 9034 : if (type.isHostVisible()) {
98 4517 : uint32_t ret = 16;
99 4517 : if (type.isHostCoherent()) { ret += 3; }
100 4517 : if (type.isHostCached()) { ret -= 4; }
101 4517 : return ret;
102 : }
103 4517 : return 0;
104 : break;
105 0 : case Allocator::MemHeapType::DeviceLocal:
106 0 : return 0;
107 : break;
108 : }
109 0 : break;
110 126105 : case AllocationUsage::HostTransitionDestination:
111 126105 : switch (heap.type) {
112 75663 : case Allocator::MemHeapType::HostLocal:
113 75663 : if (type.isHostVisible()) {
114 50442 : uint32_t ret = 32;
115 50442 : if (type.isHostCoherent()) { ret -= 3; }
116 50442 : if (type.isHostCached()) { ret += 4; }
117 50442 : return ret;
118 : }
119 25221 : return 0;
120 : break;
121 50442 : case Allocator::MemHeapType::DeviceLocalHostVisible:
122 50442 : if (type.isHostVisible()) {
123 25221 : uint32_t ret = 16;
124 25221 : if (type.isHostCoherent()) { ret -= 3; }
125 25221 : if (type.isHostCached()) { ret += 4; }
126 25221 : return ret;
127 : }
128 25221 : return 0;
129 : break;
130 0 : case Allocator::MemHeapType::DeviceLocal:
131 0 : return 0;
132 : break;
133 : }
134 0 : break;
135 : }
136 0 : return 0;
137 : }
138 :
139 84 : Allocator::~Allocator() { }
140 :
141 42 : bool Allocator::init(Device &dev, VkPhysicalDevice device, const DeviceInfo::Features &features, const DeviceInfo::Properties &props) {
142 42 : _device = &dev;
143 42 : _bufferImageGranularity = props.device10.properties.limits.bufferImageGranularity;
144 42 : _nonCoherentAtomSize = props.device10.properties.limits.nonCoherentAtomSize;
145 :
146 42 : if ((features.flags & ExtensionFlags::GetMemoryRequirements2) != ExtensionFlags::None) {
147 42 : _hasMemReq2 = true;
148 : }
149 42 : if ((features.flags & ExtensionFlags::DedicatedAllocation) != ExtensionFlags::None) {
150 42 : _hasDedicated = true;
151 : }
152 :
153 42 : if ((features.flags & ExtensionFlags::MemoryBudget) != ExtensionFlags::None) {
154 42 : _memBudget.pNext = nullptr;
155 42 : _memProperties.pNext = &_memBudget;
156 42 : _hasBudget = true;
157 : } else {
158 0 : _memProperties.pNext = nullptr;
159 : }
160 :
161 42 : dev.getInstance()->vkGetPhysicalDeviceMemoryProperties2KHR(device, &_memProperties);
162 :
163 126 : for (uint32_t i = 0; i < _memProperties.memoryProperties.memoryHeapCount; ++ i) {
164 84 : auto &it = _memHeaps.emplace_back(MemHeap{i, _memProperties.memoryProperties.memoryHeaps[i]});
165 84 : if ((it.heap.flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0) {
166 42 : it.type = DeviceLocal;
167 : }
168 504 : for (uint32_t j = 0; j < _memProperties.memoryProperties.memoryTypeCount; ++ j) {
169 420 : if (_memProperties.memoryProperties.memoryTypes[j].heapIndex == i) {
170 210 : it.types.emplace_back(MemType{j, _memProperties.memoryProperties.memoryTypes[j]});
171 210 : if ((_memProperties.memoryProperties.memoryTypes[j].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) {
172 126 : if (it.type == DeviceLocal) {
173 42 : it.type = DeviceLocalHostVisible;
174 : }
175 : }
176 : }
177 : }
178 :
179 84 : if (_hasBudget) {
180 84 : it.budget = _memBudget.heapBudget[i];
181 84 : it.usage = _memBudget.heapUsage[i];
182 : }
183 : }
184 :
185 126 : for (auto &it : _memHeaps) {
186 294 : for (auto &jt : it.types) {
187 210 : _memTypes.emplace_back(&jt);
188 : }
189 : }
190 :
191 42 : std::sort(_memTypes.begin(), _memTypes.end(), [] (const MemType *l, const MemType *r) {
192 378 : return l->idx < r->idx;
193 : });
194 :
195 : if constexpr (s_printVkInfo) {
196 42 : StringStream stream;
197 42 : stream << "[Memory]\n";
198 126 : for (auto &it : _memHeaps) {
199 84 : stream << "\t[Heap] " << it.idx << ": " << it.heap.size << " bytes;";
200 84 : if (_hasBudget) {
201 84 : stream << " Budget: " << it.budget << "; Usage: " << it.usage << ";";
202 : }
203 84 : if ((it.heap.flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0) {
204 42 : stream << " DeviceLocal;";
205 : }
206 84 : if ((it.heap.flags & VK_MEMORY_HEAP_MULTI_INSTANCE_BIT) != 0) {
207 0 : stream << " MultiInstance;";
208 : }
209 84 : stream << "\n";
210 294 : for (auto &jt : it.types) {
211 210 : stream << "\t\t[Type] " << jt.idx;
212 210 : if ((jt.type.propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0) {
213 84 : stream << " DeviceLocal;";
214 : }
215 210 : if ((jt.type.propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) {
216 126 : stream << " HostVisible;";
217 : }
218 210 : if ((jt.type.propertyFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0) {
219 126 : stream << " HostCoherent;";
220 : }
221 210 : if ((jt.type.propertyFlags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0) {
222 42 : stream << " HostCached;";
223 : }
224 210 : if ((jt.type.propertyFlags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0) {
225 0 : stream << " LazilyAllocated;";
226 : }
227 210 : if ((jt.type.propertyFlags & VK_MEMORY_PROPERTY_PROTECTED_BIT) != 0) {
228 0 : stream << " Protected;";
229 : }
230 210 : if ((jt.type.propertyFlags & VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD) != 0) {
231 0 : stream << " DeviceCoherent;";
232 : }
233 210 : if ((jt.type.propertyFlags & VK_MEMORY_PROPERTY_DEVICE_UNCACHED_BIT_AMD) != 0) {
234 0 : stream << " DeviceUncached;";
235 : }
236 210 : stream << "\n";
237 : }
238 : }
239 42 : log::verbose("Vk-Info", stream.str());
240 42 : }
241 :
242 42 : return true;
243 : }
244 :
245 42 : void Allocator::invalidate(Device &dev) {
246 252 : for (auto &type : _memTypes) {
247 4410 : for (auto &nodes : type->buf) {
248 4389 : for (auto &node : nodes) {
249 189 : if (node.ptr) {
250 126 : _device->getTable()->vkUnmapMemory(_device->getDevice(), node.mem);
251 126 : const_cast<MemNode &>(node).ptr = nullptr;
252 : }
253 189 : _device->makeApiCall([&] (const DeviceTable &table, VkDevice device) {
254 189 : table.vkFreeMemory(device, node.mem, nullptr);
255 189 : });
256 189 : const_cast<MemNode &>(node).mem = VK_NULL_HANDLE;
257 : }
258 : }
259 : }
260 :
261 42 : _device = nullptr;
262 42 : }
263 :
264 0 : void Allocator::update() {
265 0 : if (_device && _physicalDevice && _hasBudget) {
266 0 : _memBudget.pNext = nullptr;
267 0 : _memProperties.pNext = &_memBudget;
268 0 : _device->getInstance()->vkGetPhysicalDeviceMemoryProperties2KHR(_physicalDevice, &_memProperties);
269 :
270 0 : for (uint32_t i = 0; i < _memProperties.memoryProperties.memoryHeapCount; ++ i) {
271 0 : auto &it = _memHeaps[i];
272 0 : it.budget = _memBudget.heapBudget[i];
273 0 : it.usage = _memBudget.heapUsage[i];
274 : }
275 : }
276 0 : }
277 :
278 1234 : uint32_t Allocator::getInitialTypeMask() const {
279 1234 : uint32_t ret = 0;
280 7404 : for (size_t i = 0; i < _memProperties.memoryProperties.memoryTypeCount; ++ i) {
281 6170 : ret |= 1 << i;
282 : }
283 1234 : return ret;
284 : }
285 :
286 517630 : const Allocator::MemType *Allocator::getType(uint32_t idx) const {
287 517630 : if (idx < _memTypes.size()) {
288 517779 : return _memTypes[idx];
289 : }
290 0 : return nullptr;
291 : }
292 :
293 0 : void Allocator::lock() {
294 0 : _mutex.lock();
295 0 : }
296 :
297 0 : void Allocator::unlock() {
298 0 : _mutex.unlock();
299 0 : }
300 :
301 90855 : Allocator::MemNode Allocator::alloc(MemType *type, uint64_t in_size, bool persistent) {
302 90855 : std::unique_lock<Mutex> lock;
303 :
304 : // PageSize boundary should be large enough to match all alignment requirements
305 90855 : uint64_t size = uint64_t(math::align<uint64_t>(in_size, PageSize));
306 90855 : if (size < in_size) {
307 0 : return MemNode();
308 : }
309 :
310 90855 : if (size < type->min * PageSize) {
311 89768 : size = type->min * PageSize;
312 : }
313 :
314 90855 : uint64_t index = size / PageSize - type->min + 1;
315 :
316 : /* First see if there are any nodes in the area we know
317 : * our node will fit into. */
318 90855 : if (index <= type->last) {
319 90668 : lock = std::unique_lock<Mutex>(_mutex);
320 : /* Walk the free list to see if there are
321 : * any nodes on it of the requested size */
322 90668 : uint64_t max_index = type->last;
323 90668 : auto ref = &type->buf[index];
324 90668 : uint64_t i = index;
325 90668 : while (type->buf[i].empty() && i < max_index) {
326 0 : ref++;
327 0 : i++;
328 : }
329 :
330 90668 : if (!ref->empty()) {
331 90666 : auto node = ref->back();
332 90666 : ref->pop_back();
333 :
334 90666 : if (ref->empty()) {
335 : // revert `last` value
336 : do {
337 88434 : ref --;
338 88434 : max_index --;
339 88434 : } while (max_index > 0 && ref->empty());
340 88434 : type->last = max_index;
341 : }
342 :
343 90666 : type->current += node.index + (type->min - 1);
344 90666 : if (type->current > type->max) {
345 0 : type->current = type->max;
346 : }
347 :
348 90666 : if (persistent && !node.ptr) {
349 0 : if (_device->getTable()->vkMapMemory(_device->getDevice(), node.mem, 0, node.size, 0, &node.ptr) != VK_SUCCESS) {
350 0 : return MemNode();
351 : }
352 90666 : } else if (!persistent && node.ptr) {
353 0 : _device->getTable()->vkUnmapMemory(_device->getDevice(), node.mem);
354 0 : node.ptr = nullptr;
355 : }
356 :
357 90666 : return node;
358 : }
359 187 : } else if (!type->buf[0].empty()) {
360 0 : lock = std::unique_lock<Mutex>(_mutex);
361 : /* If we found nothing, seek the sink (at index 0), if
362 : * it is not empty. */
363 :
364 : /* Walk the free list to see if there are
365 : * any nodes on it of the requested size */
366 :
367 0 : auto ref = SpanView<MemNode>(type->buf[0]);
368 0 : auto it = ref.begin();
369 :
370 0 : while (it != ref.end() && index > it->index) {
371 0 : ++ it;
372 : }
373 :
374 0 : if (it != ref.end()) {
375 0 : MemNode node = *it;
376 0 : type->buf[0].erase(type->buf[0].begin() + (it - ref.begin()));
377 0 : type->current += node.index + (type->min - 1);
378 0 : if (type->current > type->max) {
379 0 : type->current = type->max;
380 : }
381 :
382 0 : if (persistent && !node.ptr) {
383 0 : if (_device->getTable()->vkMapMemory(_device->getDevice(), node.mem, 0, node.size, 0, &node.ptr) != VK_SUCCESS) {
384 0 : return MemNode();
385 : }
386 0 : } else if (!persistent && node.ptr) {
387 0 : _device->getTable()->vkUnmapMemory(_device->getDevice(), node.mem);
388 0 : node.ptr = nullptr;
389 : }
390 :
391 0 : return node;
392 : }
393 : }
394 :
395 : /* If we haven't got a suitable node, malloc a new one
396 : * and initialize it. */
397 189 : if (lock.owns_lock()) {
398 2 : lock.unlock();
399 : }
400 :
401 189 : MemNode ret;
402 189 : VkMemoryAllocateInfo allocInfo{};
403 189 : allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
404 189 : allocInfo.allocationSize = size;
405 189 : allocInfo.memoryTypeIndex = type->idx;
406 :
407 189 : VkResult result = VK_ERROR_UNKNOWN;
408 189 : _device->makeApiCall([&] (const DeviceTable &table, VkDevice device) {
409 189 : result = table.vkAllocateMemory(device, &allocInfo, nullptr, &ret.mem);
410 189 : });
411 :
412 189 : if (result != VK_SUCCESS) {
413 0 : return MemNode();
414 : }
415 :
416 189 : if (persistent && (type->type.propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) {
417 126 : if (_device->getTable()->vkMapMemory(_device->getDevice(), ret.mem, 0, size, 0, &ret.ptr) != VK_SUCCESS) {
418 0 : _device->makeApiCall([&] (const DeviceTable &table, VkDevice device) {
419 0 : table.vkFreeMemory(device, ret.mem, nullptr);
420 0 : });
421 0 : return MemNode();
422 : }
423 : }
424 :
425 189 : ret.index = index;
426 189 : ret.size = size;
427 189 : ret.offset = 0;
428 189 : return ret;
429 90855 : }
430 :
431 89789 : void Allocator::free(MemType *type, SpanView<MemNode> nodes) {
432 89789 : Vector<MemNode> freelist;
433 :
434 89789 : std::unique_lock<Mutex> lock(_mutex);
435 :
436 89789 : uint64_t max_index = type->last;
437 89789 : uint64_t max_free_index = type->max;
438 89789 : uint64_t current_free_index = type->current;
439 :
440 : /* Walk the list of submitted nodes and free them one by one,
441 : * shoving them in the right 'size' buckets as we go. */
442 :
443 180644 : for (auto &node : nodes) {
444 90855 : uint64_t index = node.index;
445 :
446 : // do not store large allocations
447 90855 : if (max_free_index != maxOf<uint64_t>() && index + (type->min - 1) > current_free_index) {
448 0 : freelist.emplace_back(node);
449 90855 : } else if (index < MaxIndex) {
450 : /* Add the node to the appropriate 'size' bucket. Adjust
451 : * the max_index when appropriate. */
452 90855 : if (type->buf[index].empty() && index > max_index) {
453 88560 : max_index = index;
454 : }
455 90855 : type->buf[index].emplace_back(node);
456 90855 : if (current_free_index >= index + (type->min - 1)) {
457 90855 : current_free_index -= index + (type->min - 1);
458 : } else {
459 0 : current_free_index = 0;
460 : }
461 : } else {
462 : /* This node is too large to keep in a specific size bucket,
463 : * just add it to the sink (at index 0).
464 : */
465 0 : type->buf[0].emplace_back(node);
466 0 : if (current_free_index >= index + (type->min - 1)) {
467 0 : current_free_index -= index + (type->min - 1);
468 : } else {
469 0 : current_free_index = 0;
470 : }
471 : }
472 : }
473 :
474 89789 : type->last = max_index;
475 89789 : type->current = current_free_index;
476 :
477 89789 : for (const MemNode &it : freelist) {
478 0 : if (it.ptr) {
479 0 : _device->getTable()->vkUnmapMemory(_device->getDevice(), it.mem);
480 0 : const_cast<MemNode &>(it).ptr = nullptr;
481 : }
482 0 : _device->makeApiCall([&] (const DeviceTable &table, VkDevice device) {
483 0 : table.vkFreeMemory(device, it.mem, nullptr);
484 0 : });
485 : }
486 89789 : }
487 :
488 698460 : Allocator::MemType * Allocator::findMemoryType(uint32_t typeFilter, AllocationUsage type) const {
489 : // best match
490 698460 : uint32_t score = 0;
491 698460 : uint32_t idx = maxOf<uint32_t>();
492 4190444 : for (auto &it : _memTypes) {
493 3492016 : if ((typeFilter & (1 << it->idx))) {
494 3488135 : auto s = Allocator_getTypeScoreInternal(_memHeaps[it->type.heapIndex], *it, type);
495 3488103 : if (s && s > score) {
496 828397 : score = s;
497 828397 : idx = it->idx;
498 : }
499 : }
500 : }
501 :
502 698334 : if (idx != maxOf<uint32_t>()) {
503 698445 : return (Allocator::MemType *)_memTypes[idx];
504 : }
505 :
506 0 : auto getTypeName = [&] (AllocationUsage t) {
507 0 : switch (t) {
508 0 : case AllocationUsage::DeviceLocal: return StringView("DeviceLocal"); break;
509 0 : case AllocationUsage::DeviceLocalHostVisible: return StringView("DeviceLocalHostVisible"); break;
510 0 : case AllocationUsage::HostTransitionDestination: return StringView("HostTransitionDestination"); break;
511 0 : case AllocationUsage::HostTransitionSource: return StringView("HostTransitionSource"); break;
512 0 : default: break;
513 : }
514 0 : return StringView("Unknown");
515 : };
516 :
517 0 : log::error("Vk-Error", "Fail to find required memory type for ", getTypeName(type));
518 0 : return nullptr;
519 : }
520 :
521 699717 : MemoryRequirements Allocator::getBufferMemoryRequirements(VkBuffer target) {
522 699717 : MemoryRequirements ret;
523 699717 : VkMemoryRequirements2 memRequirements = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2, nullptr };
524 :
525 699717 : if (hasMemReq2Feature() && hasDedicatedFeature()) {
526 699717 : VkMemoryDedicatedRequirements memDedicatedReqs = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS };
527 699717 : memRequirements.pNext = &memDedicatedReqs;
528 :
529 699717 : VkBufferMemoryRequirementsInfo2 info = { VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2, nullptr, target };
530 699717 : _device->getTable()->vkGetBufferMemoryRequirements2(_device->getDevice(), &info, &memRequirements);
531 :
532 699719 : ret.requiresDedicated = memDedicatedReqs.requiresDedicatedAllocation;
533 699719 : ret.prefersDedicated = memDedicatedReqs.prefersDedicatedAllocation;
534 699719 : ret.requirements = memRequirements.memoryRequirements;
535 : } else {
536 0 : _device->getTable()->vkGetBufferMemoryRequirements(_device->getDevice(), target, &ret.requirements);
537 : }
538 1399438 : return ret;
539 : }
540 :
541 1360 : MemoryRequirements Allocator::getImageMemoryRequirements(VkImage target) {
542 1360 : MemoryRequirements ret;
543 1360 : VkMemoryRequirements2 memRequirements = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2, nullptr };
544 :
545 1360 : if (hasMemReq2Feature() && hasDedicatedFeature()) {
546 1360 : VkMemoryDedicatedRequirements memDedicatedReqs = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS, nullptr };
547 1360 : memRequirements.pNext = &memDedicatedReqs;
548 :
549 1360 : VkImageMemoryRequirementsInfo2 info = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2, nullptr, target };
550 1360 : _device->getTable()->vkGetImageMemoryRequirements2(_device->getDevice(), &info, &memRequirements);
551 :
552 1360 : ret.requiresDedicated = memDedicatedReqs.requiresDedicatedAllocation;
553 1360 : ret.prefersDedicated = memDedicatedReqs.prefersDedicatedAllocation;
554 1360 : ret.requirements = memRequirements.memoryRequirements;
555 : } else {
556 0 : _device->getTable()->vkGetImageMemoryRequirements(_device->getDevice(), target, &ret.requirements);
557 : }
558 2720 : return ret;
559 : }
560 :
561 1256 : Rc<Buffer> Allocator::spawnPersistent(AllocationUsage usage, const BufferInfo &info, BytesView view) {
562 1256 : auto target = preallocate(info, view);
563 1256 : if (!target) {
564 0 : return nullptr;
565 : }
566 :
567 1256 : if (!allocateDedicated(usage, target)) {
568 0 : return nullptr;
569 : }
570 :
571 1256 : if (!view.empty()) {
572 42 : void *ptr = nullptr;
573 42 : if (_device->getTable()->vkMapMemory(_device->getDevice(), target->getMemory()->getMemory(), 0, view.size(), 0, &ptr) == VK_SUCCESS) {
574 42 : memcpy(ptr, view.data(), view.size());
575 42 : _device->getTable()->vkUnmapMemory(_device->getDevice(), target->getMemory()->getMemory());
576 : } else {
577 0 : return nullptr;
578 : }
579 : }
580 :
581 1256 : return target;
582 1256 : }
583 :
584 147 : Rc<Image> Allocator::spawnPersistent(AllocationUsage usage, const core::ImageInfoData &info, bool preinitialized, uint64_t forceId) {
585 147 : auto target = preallocate(info, preinitialized, forceId);
586 147 : if (!target) {
587 0 : return nullptr;
588 : }
589 :
590 147 : if (!allocateDedicated(usage, target)) {
591 0 : return nullptr;
592 : }
593 :
594 147 : return target;
595 147 : }
596 :
597 699237 : Rc<Buffer> Allocator::preallocate(const BufferInfo &info, BytesView view) {
598 699237 : VkBufferCreateInfo bufferInfo { };
599 699237 : bufferInfo.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO;
600 699237 : bufferInfo.size = (view.empty() ? info.size : view.size());
601 699236 : bufferInfo.flags = VkBufferCreateFlags(info.flags);
602 699236 : bufferInfo.usage = VkBufferUsageFlags(info.usage);
603 699236 : bufferInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
604 :
605 699236 : VkBuffer target = VK_NULL_HANDLE;
606 699236 : if (_device->getTable()->vkCreateBuffer(_device->getDevice(), &bufferInfo, nullptr, &target) != VK_SUCCESS) {
607 0 : return nullptr;
608 : }
609 :
610 1398513 : return Rc<Buffer>::create(*_device, target, info, nullptr, 0);
611 : }
612 :
613 1297 : Rc<Image> Allocator::preallocate(const ImageInfoData &info, bool preinitialized, uint64_t forceId) {
614 1297 : VkImageCreateInfo imageInfo { };
615 1297 : imageInfo.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO;
616 1297 : imageInfo.pNext = nullptr;
617 1297 : imageInfo.flags = info.flags;
618 1297 : imageInfo.imageType = VkImageType(info.imageType);
619 1297 : imageInfo.format = VkFormat(info.format);
620 1297 : imageInfo.extent = VkExtent3D({ info.extent.width, info.extent.height, info.extent.depth });
621 1297 : imageInfo.mipLevels = info.mipLevels.get();
622 1297 : imageInfo.arrayLayers = info.arrayLayers.get();
623 1297 : imageInfo.samples = VkSampleCountFlagBits(info.samples);
624 1297 : imageInfo.tiling = VkImageTiling(info.tiling);
625 1297 : imageInfo.usage = VkImageUsageFlags(info.usage);
626 1297 : imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
627 :
628 1297 : if (preinitialized) {
629 0 : imageInfo.initialLayout = VK_IMAGE_LAYOUT_PREINITIALIZED;
630 : } else {
631 1297 : imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
632 : }
633 :
634 1297 : VkImage target = VK_NULL_HANDLE;
635 1297 : if (_device->getTable()->vkCreateImage(_device->getDevice(), &imageInfo, nullptr, &target) != VK_SUCCESS) {
636 0 : return nullptr;
637 : }
638 :
639 1297 : if (forceId) {
640 2132 : return Rc<Image>::create(*_device, forceId, target, info, nullptr);
641 : } else {
642 462 : return Rc<Image>::create(*_device, target, info, nullptr);
643 : }
644 : }
645 :
646 1108 : Rc<DeviceMemory> Allocator::emplaceObjects(AllocationUsage usage, SpanView<Rc<Image>> images, SpanView<Rc<Buffer>> buffers) {
647 1108 : Vector<MemoryRequirements> bufferRequirements; bufferRequirements.reserve(buffers.size());
648 1108 : Vector<MemoryRequirements> imageRequirements; imageRequirements.reserve(images.size());
649 :
650 1108 : uint32_t linearObjects = 0;
651 1108 : uint32_t nonLinearObjects = 0;
652 :
653 1108 : auto mask = getInitialTypeMask();
654 :
655 3282 : for (auto &it : buffers) {
656 2174 : auto req = getBufferMemoryRequirements(it->getBuffer());
657 2174 : if (!req.prefersDedicated && !req.requiresDedicated) {
658 2174 : mask &= req.requirements.memoryTypeBits;
659 : }
660 2174 : if (mask == 0) {
661 0 : log::error("vk::Allocator", "emplaceObjects: fail to find common memory type");
662 0 : return nullptr;
663 : }
664 2174 : bufferRequirements.emplace_back(req);
665 2174 : ++ linearObjects;
666 : }
667 :
668 2258 : for (auto &it : images) {
669 1150 : auto req = getImageMemoryRequirements(it->getImage());
670 1150 : if (!req.prefersDedicated && !req.requiresDedicated) {
671 1150 : mask &= req.requirements.memoryTypeBits;
672 : }
673 1150 : if (mask == 0) {
674 0 : log::error("vk::Allocator", "emplaceObjects: fail to find common memory type");
675 0 : return nullptr;
676 : }
677 1150 : imageRequirements.emplace_back(req);
678 :
679 1150 : if (it->getInfo().tiling == core::ImageTiling::Optimal) {
680 1150 : ++ nonLinearObjects;
681 : } else {
682 0 : ++ linearObjects;
683 : }
684 : }
685 :
686 1108 : auto allocMemType = findMemoryType(mask, usage);
687 1108 : if (!allocMemType) {
688 0 : log::error("vk::Allocator", "emplaceObjects: fail to find memory type");
689 0 : return nullptr;
690 : }
691 :
692 1108 : VkDeviceSize nonCoherentAtomSize = 1;
693 1108 : if (allocMemType->isHostVisible()) {
694 0 : if (!allocMemType->isHostCoherent()) {
695 0 : nonCoherentAtomSize = getNonCoherentAtomSize();
696 : }
697 : }
698 :
699 1108 : VkDeviceSize requiredMemory = 0;
700 :
701 1108 : uint32_t i = 0;
702 1108 : if (nonLinearObjects > 0) {
703 2258 : for (auto &it : images) {
704 1150 : if (!imageRequirements[i].requiresDedicated && !imageRequirements[i].prefersDedicated) {
705 1150 : if (it->getInfo().tiling == core::ImageTiling::Optimal) {
706 1150 : requiredMemory = math::align<VkDeviceSize>(requiredMemory,
707 1150 : std::max(imageRequirements[i].requirements.alignment, nonCoherentAtomSize));
708 1150 : imageRequirements[i].targetOffset = requiredMemory;
709 1150 : requiredMemory += imageRequirements[i].requirements.size;
710 : }
711 : }
712 1150 : ++ i;
713 : }
714 : }
715 :
716 1108 : if (nonLinearObjects > 0 && linearObjects > 0) {
717 1108 : requiredMemory = math::align<VkDeviceSize>(requiredMemory, getBufferImageGranularity());
718 : }
719 :
720 1108 : if (linearObjects) {
721 1108 : i = 0;
722 2258 : for (auto &it : images) {
723 1150 : if (!imageRequirements[i].requiresDedicated && !imageRequirements[i].prefersDedicated) {
724 1150 : if (it->getInfo().tiling != core::ImageTiling::Optimal) {
725 0 : requiredMemory = math::align<VkDeviceSize>(requiredMemory,
726 0 : std::max(imageRequirements[i].requirements.alignment, nonCoherentAtomSize));
727 0 : imageRequirements[i].targetOffset = requiredMemory;
728 0 : requiredMemory += imageRequirements[i].requirements.size;
729 : }
730 : }
731 1150 : ++ i;
732 : }
733 :
734 1108 : i = 0;
735 3282 : for (auto &it : bufferRequirements) {
736 2174 : if (!it.requiresDedicated && !it.prefersDedicated) {
737 2174 : requiredMemory = math::align<VkDeviceSize>(requiredMemory,
738 2174 : std::max(it.requirements.alignment, nonCoherentAtomSize));
739 2174 : it.targetOffset = requiredMemory;
740 2174 : requiredMemory += it.requirements.size;
741 : }
742 2174 : ++ i;
743 : }
744 : }
745 :
746 : VkDeviceMemory memObject;
747 1108 : if (requiredMemory > 0) {
748 1108 : VkMemoryAllocateInfo allocInfo{};
749 1108 : allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
750 1108 : allocInfo.allocationSize = requiredMemory;
751 1108 : allocInfo.memoryTypeIndex = allocMemType->idx;
752 :
753 1108 : if (_device->getTable()->vkAllocateMemory(_device->getDevice(), &allocInfo, nullptr, &memObject) != VK_SUCCESS) {
754 0 : log::error("vk::Allocator", "emplaceObjects: fail to allocate memory");
755 0 : return nullptr;
756 : }
757 : }
758 :
759 0 : auto memory = Rc<DeviceMemory>::create(this, DeviceMemoryInfo{
760 1108 : requiredMemory, 1, allocMemType->idx, false
761 1108 : }, memObject, usage);
762 :
763 : // bind memory
764 1108 : if (nonLinearObjects > 0) {
765 1108 : i = 0;
766 2258 : for (auto &it : images) {
767 1150 : if (imageRequirements[i].requiresDedicated || imageRequirements[i].prefersDedicated) {
768 0 : if (!allocateDedicated(usage, it)) {
769 0 : return nullptr;
770 : }
771 : } else {
772 1150 : if (it->getInfo().tiling == core::ImageTiling::Optimal) {
773 1150 : it->bindMemory(Rc<DeviceMemory>(memory), imageRequirements[i].targetOffset);
774 : }
775 : }
776 1150 : ++ i;
777 : }
778 : }
779 :
780 1108 : if (linearObjects > 0) {
781 1108 : i = 0;
782 2258 : for (auto &it : images) {
783 1150 : if (!imageRequirements[i].requiresDedicated && !imageRequirements[i].prefersDedicated) {
784 1150 : if (it->getInfo().tiling != core::ImageTiling::Optimal) {
785 0 : it->bindMemory(Rc<DeviceMemory>(memory), imageRequirements[i].targetOffset);
786 : }
787 : }
788 1150 : ++ i;
789 : }
790 :
791 1108 : i = 0;
792 3282 : for (auto &it : buffers) {
793 2174 : if (bufferRequirements[i].requiresDedicated || bufferRequirements[i].prefersDedicated) {
794 0 : if (!allocateDedicated(usage, it)) {
795 0 : return nullptr;
796 : }
797 : } else {
798 2174 : it->bindMemory(Rc<DeviceMemory>(memory), bufferRequirements[i].targetOffset);
799 : }
800 2174 : ++ i;
801 : }
802 : }
803 1108 : return memory;
804 1108 : }
805 :
806 1256 : bool Allocator::allocateDedicated(AllocationUsage usage, Buffer *target) {
807 1256 : auto req = getBufferMemoryRequirements(target->getBuffer());
808 1256 : auto type = findMemoryType(req.requirements.memoryTypeBits, usage);
809 1256 : if (!type) {
810 0 : log::error("vk::Allocator", "Buffer: allocateDedicated: Fail to find memory type");
811 0 : return false;
812 : }
813 :
814 : VkDeviceMemory memory;
815 1256 : if (hasDedicatedFeature()) {
816 : VkMemoryDedicatedAllocateInfo dedicatedInfo;
817 1256 : dedicatedInfo.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO;
818 1256 : dedicatedInfo.pNext = nullptr;
819 1256 : dedicatedInfo.image = VK_NULL_HANDLE;
820 1256 : dedicatedInfo.buffer = target->getBuffer();
821 :
822 1256 : VkMemoryAllocateInfo allocInfo{};
823 1256 : allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
824 1256 : allocInfo.pNext = &dedicatedInfo;
825 1256 : allocInfo.allocationSize = req.requirements.size;
826 1256 : allocInfo.memoryTypeIndex = type->idx;
827 :
828 1256 : VkResult result = VK_ERROR_UNKNOWN;
829 1256 : _device->makeApiCall([&] (const DeviceTable &table, VkDevice device) {
830 1256 : result = table.vkAllocateMemory(device, &allocInfo, nullptr, &memory);
831 1256 : });
832 1256 : if (result != VK_SUCCESS) {
833 0 : return false;
834 : }
835 : } else {
836 0 : VkMemoryAllocateInfo allocInfo{};
837 0 : allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
838 0 : allocInfo.pNext = nullptr;
839 0 : allocInfo.allocationSize = req.requirements.size;
840 0 : allocInfo.memoryTypeIndex = type->idx;
841 :
842 0 : VkResult result = VK_ERROR_UNKNOWN;
843 0 : _device->makeApiCall([&] (const DeviceTable &table, VkDevice device) {
844 0 : result = table.vkAllocateMemory(device, &allocInfo, nullptr, &memory);
845 0 : });
846 0 : if (result != VK_SUCCESS) {
847 0 : return false;
848 : }
849 : }
850 :
851 1256 : target->bindMemory(Rc<DeviceMemory>::create(this, DeviceMemoryInfo{
852 1256 : req.requirements.size, req.requirements.alignment, type->idx, true
853 : }, memory, usage));
854 1256 : return true;
855 : }
856 :
857 147 : bool Allocator::allocateDedicated(AllocationUsage usage, Image *target) {
858 147 : auto req = getImageMemoryRequirements(target->getImage());
859 147 : auto type = findMemoryType(req.requirements.memoryTypeBits, usage);
860 147 : if (!type) {
861 0 : log::error("vk::Allocator", "Image: allocateDedicated: Fail to find memory type");
862 0 : return false;
863 : }
864 :
865 : VkDeviceMemory memory;
866 147 : if (hasDedicatedFeature()) {
867 : VkMemoryDedicatedAllocateInfo dedicatedInfo;
868 147 : dedicatedInfo.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO;
869 147 : dedicatedInfo.pNext = nullptr;
870 147 : dedicatedInfo.image = target->getImage();
871 147 : dedicatedInfo.buffer = VK_NULL_HANDLE;
872 :
873 147 : VkMemoryAllocateInfo allocInfo{};
874 147 : allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
875 147 : allocInfo.pNext = &dedicatedInfo;
876 147 : allocInfo.allocationSize = req.requirements.size;
877 147 : allocInfo.memoryTypeIndex = type->idx;
878 :
879 147 : VkResult result = VK_ERROR_UNKNOWN;
880 147 : _device->makeApiCall([&] (const DeviceTable &table, VkDevice device) {
881 147 : result = table.vkAllocateMemory(device, &allocInfo, nullptr, &memory);
882 147 : });
883 147 : if (result != VK_SUCCESS) {
884 0 : log::error("vk::Allocator", "Image: allocateDedicated: Fail to allocate memory for dedicated allocation");
885 0 : return false;
886 : }
887 : } else {
888 0 : VkMemoryAllocateInfo allocInfo{};
889 0 : allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
890 0 : allocInfo.pNext = nullptr;
891 0 : allocInfo.allocationSize = req.requirements.size;
892 0 : allocInfo.memoryTypeIndex = type->idx;
893 :
894 0 : VkResult result = VK_ERROR_UNKNOWN;
895 0 : _device->makeApiCall([&] (const DeviceTable &table, VkDevice device) {
896 0 : result = table.vkAllocateMemory(device, &allocInfo, nullptr, &memory);
897 0 : });
898 0 : if (result != VK_SUCCESS) {
899 0 : log::error("vk::Allocator", "Image: allocateDedicated: Fail to allocate memory for dedicated allocation");
900 0 : return false;
901 : }
902 : }
903 :
904 147 : target->bindMemory(Rc<DeviceMemory>::create(this, DeviceMemoryInfo{
905 147 : req.requirements.size, req.requirements.alignment, type->idx, true
906 : }, memory, usage));
907 147 : return true;
908 : }
909 :
910 124522 : DeviceMemoryPool::~DeviceMemoryPool() {
911 153116 : for (auto &it : _mappingProtection) {
912 90855 : delete it.second;
913 : }
914 :
915 62261 : if (_allocator) {
916 758088 : for (auto &it : _buffers) {
917 695827 : it->invalidate();
918 : }
919 62261 : _buffers.clear();
920 62261 : for (auto &it : _images) {
921 0 : it->invalidate();
922 : }
923 62261 : _images.clear();
924 152050 : for (auto &it : _heaps) {
925 89789 : clear(&it.second);
926 : }
927 : }
928 124522 : }
929 :
930 62261 : bool DeviceMemoryPool::init(const Rc<Allocator> &alloc, bool persistentMapping) {
931 62261 : _allocator = alloc;
932 62261 : _persistentMapping = persistentMapping;
933 62261 : return true;
934 : }
935 :
936 695810 : Rc<Buffer> DeviceMemoryPool::spawn(AllocationUsage type, const BufferInfo &info) {
937 695810 : auto buffer = _allocator->preallocate(info);
938 695826 : auto requirements = _allocator->getBufferMemoryRequirements(buffer->getBuffer());
939 :
940 695826 : if (requirements.requiresDedicated) {
941 : // TODO: deal with dedicated allocations
942 0 : log::error("DeviceMemoryPool", "Dedicated allocation required");
943 : } else {
944 695826 : auto memType = _allocator->findMemoryType(requirements.requirements.memoryTypeBits, type);
945 695808 : if (!memType) {
946 0 : return nullptr;
947 : }
948 :
949 695808 : std::unique_lock<Mutex> lock(_mutex);
950 695827 : MemData *pool = nullptr;
951 695827 : auto it = _heaps.find(memType->idx);
952 695827 : if (it == _heaps.end()) {
953 89789 : pool = &_heaps.emplace(memType->idx, MemData{memType}).first->second;
954 : } else {
955 606038 : pool = &it->second;
956 : }
957 :
958 1391654 : if (auto mem = alloc(pool, requirements.requirements.size,
959 695827 : requirements.requirements.alignment, AllocationType::Linear, type)) {
960 695827 : if (buffer->bindMemory(Rc<DeviceMemory>::create(this, move(mem), type))) {
961 695827 : _buffers.emplace_front(buffer);
962 695827 : return buffer;
963 : } else {
964 0 : log::error("DeviceMemoryPool", "Fail to bind memory for buffer: ", type);
965 : }
966 : } else {
967 0 : log::error("DeviceMemoryPool", "Fail to allocate memory for buffer: ", type);
968 : }
969 695827 : }
970 0 : return nullptr;
971 695827 : }
972 :
973 0 : Rc<Image> DeviceMemoryPool::spawn(AllocationUsage type, const ImageInfoData &data) {
974 0 : auto image = _allocator->preallocate(data, false);
975 0 : auto requirements = _allocator->getImageMemoryRequirements(image->getImage());
976 :
977 0 : if (requirements.requiresDedicated) {
978 : // TODO: deal with dedicated allocations
979 0 : log::error("DeviceMemoryPool", "Dedicated allocation required");
980 : } else {
981 0 : auto memType = _allocator->findMemoryType(requirements.requirements.memoryTypeBits, type);
982 0 : if (!memType) {
983 0 : return nullptr;
984 : }
985 :
986 0 : std::unique_lock<Mutex> lock(_mutex);
987 0 : MemData *pool = nullptr;
988 0 : auto it = _heaps.find(memType->idx);
989 0 : if (it == _heaps.end()) {
990 0 : pool = &_heaps.emplace(memType->idx, MemData{memType}).first->second;
991 : } else {
992 0 : pool = &it->second;
993 : }
994 :
995 0 : if (auto mem = alloc(pool, requirements.requirements.size, requirements.requirements.alignment,
996 0 : (data.tiling == core::ImageTiling::Optimal) ? AllocationType::Optimal : AllocationType::Linear,
997 0 : type)) {
998 0 : if (image->bindMemory(Rc<DeviceMemory>::create(this, move(mem), type))) {
999 0 : _images.emplace_front(image);
1000 0 : return image;
1001 : } else {
1002 0 : log::error("DeviceMemoryPool", "Fail to bind memory for image");
1003 : }
1004 : } else {
1005 0 : log::error("DeviceMemoryPool", "Fail to allocate memory for image: ", type);
1006 : }
1007 0 : }
1008 :
1009 0 : return nullptr;
1010 0 : }
1011 :
1012 1214 : Rc<Buffer> DeviceMemoryPool::spawnPersistent(AllocationUsage usage, const BufferInfo &info) {
1013 1214 : return _allocator->spawnPersistent(usage, info);
1014 : }
1015 :
1016 0 : Device *DeviceMemoryPool::getDevice() const {
1017 0 : return _allocator->getDevice();
1018 : }
1019 :
1020 695827 : Allocator::MemBlock DeviceMemoryPool::alloc(MemData *mem, VkDeviceSize in_size, VkDeviceSize alignment,
1021 : AllocationType allocType, AllocationUsage type) {
1022 695827 : if (allocType == AllocationType::Unknown) {
1023 0 : return Allocator::MemBlock();
1024 : }
1025 :
1026 695827 : auto size = math::align<VkDeviceSize>(in_size, alignment);
1027 :
1028 695827 : Allocator::MemNode *node = nullptr;
1029 :
1030 695827 : size_t alignedOffset = 0;
1031 697959 : for (auto &it : mem->mem) {
1032 607104 : alignedOffset = math::align<VkDeviceSize>(it.offset, alignment);
1033 :
1034 607104 : if (mem->type->isHostVisible() && !mem->type->isHostCoherent()) {
1035 0 : alignedOffset = math::align<VkDeviceSize>(alignedOffset, _allocator->getNonCoherentAtomSize());
1036 : }
1037 :
1038 607104 : if (it.lastAllocation != allocType && it.lastAllocation != AllocationType::Unknown) {
1039 0 : alignedOffset = math::align<VkDeviceSize>(alignedOffset, _allocator->getBufferImageGranularity());
1040 : }
1041 :
1042 607104 : if (alignedOffset + size < it.size) {
1043 604972 : node = ⁢
1044 604972 : break;
1045 : }
1046 : }
1047 :
1048 695827 : if (!node) {
1049 90855 : size_t reqSize = size;
1050 90855 : auto b = _allocator->alloc(mem->type, reqSize, (type == AllocationUsage::DeviceLocal) ? false : _persistentMapping);
1051 90855 : mem->mem.emplace_back(b);
1052 90855 : node = &mem->mem.back();
1053 90855 : node->mappingProtection = _mappingProtection.emplace(node->mem, new Mutex()).first->second;
1054 90855 : alignedOffset = 0;
1055 : }
1056 :
1057 695827 : if (node && node->mem) {
1058 695827 : node->offset = alignedOffset + size;
1059 695827 : node->lastAllocation = allocType;
1060 695827 : return Allocator::MemBlock({node->mem, alignedOffset, size, mem->type->idx, node->ptr, node->mappingProtection});
1061 : }
1062 :
1063 0 : return Allocator::MemBlock();
1064 : }
1065 :
1066 695827 : void DeviceMemoryPool::free(Allocator::MemBlock &&block) {
1067 695827 : auto it = _heaps.find(block.type);
1068 695827 : if (it != _heaps.end()) {
1069 695827 : it->second.freed.emplace_back(move(block));
1070 : }
1071 695827 : }
1072 :
1073 89789 : void DeviceMemoryPool::clear(MemData *mem) {
1074 89789 : _allocator->free(mem->type, mem->mem);
1075 89789 : mem->mem.clear();
1076 89789 : mem->freed.clear();
1077 89789 : }
1078 :
1079 : }
|
