Line data Source code
1 : /**
2 : Copyright (c) 2021 Roman Katuntsev <sbkarr@stappler.org>
3 : Copyright (c) 2023 Stappler LLC <admin@stappler.dev>
4 :
5 : Permission is hereby granted, free of charge, to any person obtaining a copy
6 : of this software and associated documentation files (the "Software"), to deal
7 : in the Software without restriction, including without limitation the rights
8 : to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 : copies of the Software, and to permit persons to whom the Software is
10 : furnished to do so, subject to the following conditions:
11 :
12 : The above copyright notice and this permission notice shall be included in
13 : all copies or substantial portions of the Software.
14 :
15 : THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 : IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 : FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 : AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 : LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 : OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 : THE SOFTWARE.
22 : **/
23 :
24 : #include "XLVkDeviceQueue.h"
25 : #include "XLVkDevice.h"
26 : #include "XLVkSync.h"
27 : #include "XLVkObject.h"
28 : #include "XLCoreImageStorage.h"
29 : #include "XLCoreFrameQueue.h"
30 : #include "XLVkPipeline.h"
31 : #include "XLVkRenderPass.h"
32 :
33 : namespace STAPPLER_VERSIONIZED stappler::xenolith::vk {
34 :
35 840 : DeviceQueue::~DeviceQueue() { }
36 :
37 420 : bool DeviceQueue::init(Device &device, VkQueue queue, uint32_t index, QueueOperations ops) {
38 420 : _device = &device;
39 420 : _queue = queue;
40 420 : _index = index;
41 420 : _ops = ops;
42 420 : return true;
43 : }
44 :
45 172586 : bool DeviceQueue::submit(const FrameSync &sync, Fence &fence, CommandPool &commandPool, SpanView<const CommandBuffer *> buffers, IdleMode idle) {
46 172586 : Vector<VkSemaphore> waitSem;
47 172586 : Vector<VkPipelineStageFlags> waitStages;
48 172586 : Vector<VkSemaphore> signalSem;
49 172585 : Vector<VkCommandBuffer> vkBuffers; vkBuffers.reserve(buffers.size());
50 :
51 345176 : for (auto &it : buffers) {
52 172588 : if (it) {
53 172588 : vkBuffers.emplace_back(it->getBuffer());
54 : }
55 : }
56 :
57 336611 : for (auto &it : sync.waitAttachments) {
58 164026 : if (it.semaphore) {
59 164026 : auto sem = ((Semaphore *)it.semaphore.get())->getSemaphore();
60 :
61 164026 : if (!it.semaphore->isWaited()) {
62 164026 : waitSem.emplace_back(sem);
63 164026 : waitStages.emplace_back(VkPipelineStageFlags(it.stages));
64 : }
65 164026 : commandPool.autorelease(it.semaphore.get());
66 : }
67 : }
68 :
69 336686 : for (auto &it : sync.signalAttachments) {
70 164102 : if (it.semaphore) {
71 164104 : auto sem = ((Semaphore *)it.semaphore.get())->getSemaphore();
72 :
73 164104 : signalSem.emplace_back(sem);
74 164100 : commandPool.autorelease(it.semaphore.get());
75 : }
76 : }
77 :
78 172571 : VkSubmitInfo submitInfo{};
79 172571 : submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
80 172571 : submitInfo.pNext = nullptr;
81 172571 : submitInfo.waitSemaphoreCount = waitSem.size();
82 172585 : submitInfo.pWaitSemaphores = waitSem.data();
83 172585 : submitInfo.pWaitDstStageMask = waitStages.data();
84 172585 : submitInfo.commandBufferCount = vkBuffers.size();
85 172585 : submitInfo.pCommandBuffers = vkBuffers.data();
86 172585 : submitInfo.signalSemaphoreCount = signalSem.size();
87 172585 : submitInfo.pSignalSemaphores = signalSem.data();
88 :
89 : #if XL_VKAPI_DEBUG
90 : auto t = platform::clock(core::ClockType::Monotonic);
91 : fence.addRelease([frameIdx = _frameIdx, t] (bool success) {
92 : XL_VKAPI_LOG("[", frameIdx, "] vkQueueSubmit [complete]",
93 : " [", platform::clock(core::ClockType::Monotonic) - t, "]");
94 : }, nullptr, "DeviceQueue::submit");
95 : #endif
96 :
97 172586 : _device->makeApiCall([&, this] (const DeviceTable &table, VkDevice device) {
98 172586 : switch (idle) {
99 0 : case IdleMode::Queue:
100 0 : table.vkQueueWaitIdle(_queue);
101 0 : break;
102 0 : case IdleMode::Device:
103 0 : table.vkDeviceWaitIdle(_device->getDevice());
104 0 : break;
105 172586 : case IdleMode::None:
106 172586 : break;
107 : }
108 : #if XL_VKAPI_DEBUG
109 : auto t = platform::clock(core::ClockType::Monotonic);
110 : _result = table.vkQueueSubmit(_queue, 1, &submitInfo, fence.getFence());
111 : XL_VKAPI_LOG("[", _frameIdx, "] vkQueueSubmit: ", _result, " ", (void *)_queue,
112 : " [", platform::clock(core::ClockType::Monotonic) - t, "]");
113 : #else
114 172586 : _result = table.vkQueueSubmit(_queue, 1, &submitInfo, fence.getFence());
115 : #endif
116 172598 : });
117 :
118 172598 : if (_result == VK_SUCCESS) {
119 : // mark semaphores
120 :
121 336624 : for (auto &it : sync.waitAttachments) {
122 164026 : if (it.semaphore) {
123 164026 : it.semaphore->setWaited(true);
124 164026 : if (it.image && !it.image->isSemaphorePersistent()) {
125 19034 : fence.addRelease([img = it.image, sem = it.semaphore.get(), t = it.semaphore->getTimeline()] (bool success) {
126 9517 : sem->setInUse(false, t);
127 9517 : img->releaseSemaphore(sem);
128 9517 : }, it.image, "DeviceQueue::submit::!isSemaphorePersistent");
129 : } else {
130 154509 : fence.addRelease([sem = it.semaphore.get(), t = it.semaphore->getTimeline()] (bool success) {
131 154509 : sem->setInUse(false, t);
132 154509 : }, it.semaphore, "DeviceQueue::submit::isSemaphorePersistent");
133 : }
134 164026 : fence.autorelease(it.semaphore.get());
135 164026 : commandPool.autorelease(it.semaphore.get());
136 : }
137 : }
138 :
139 336707 : for (auto &it : sync.signalAttachments) {
140 164110 : if (it.semaphore) {
141 164110 : it.semaphore->setSignaled(true);
142 164110 : it.semaphore->setInUse(true, it.semaphore->getTimeline());
143 164110 : fence.autorelease(it.semaphore.get());
144 164110 : commandPool.autorelease(it.semaphore.get());
145 : }
146 : }
147 :
148 172579 : fence.setArmed(*this);
149 :
150 220201 : for (auto &it : sync.images) {
151 47606 : it.image->setLayout(it.newLayout);
152 : }
153 :
154 172595 : return true;
155 : }
156 0 : return false;
157 172595 : }
158 :
159 25305 : bool DeviceQueue::submit(Fence &fence, const CommandBuffer *buffer, IdleMode idle) {
160 25305 : return submit(fence, makeSpanView(&buffer, 1), idle);
161 : }
162 :
163 25305 : bool DeviceQueue::submit(Fence &fence, SpanView<const CommandBuffer *> buffers, IdleMode idle) {
164 25305 : Vector<VkCommandBuffer> vkBuffers; vkBuffers.reserve(buffers.size());
165 :
166 50610 : for (auto &it : buffers) {
167 25305 : if (it) {
168 25305 : vkBuffers.emplace_back(it->getBuffer());
169 : }
170 : }
171 :
172 25305 : VkSubmitInfo submitInfo{};
173 25305 : submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
174 25305 : submitInfo.pNext = nullptr;
175 25305 : submitInfo.waitSemaphoreCount = 0;
176 25305 : submitInfo.pWaitSemaphores = nullptr;
177 25305 : submitInfo.pWaitDstStageMask = 0;
178 25305 : submitInfo.commandBufferCount = vkBuffers.size();
179 25305 : submitInfo.pCommandBuffers = vkBuffers.data();
180 25305 : submitInfo.signalSemaphoreCount = 0;
181 25305 : submitInfo.pSignalSemaphores = nullptr;
182 :
183 : #if XL_VKAPI_DEBUG
184 : [[maybe_unused]] auto frameIdx = _frameIdx;
185 : [[maybe_unused]] auto t = platform::clock(core::ClockType::Monotonic);
186 : fence.addRelease([=] (bool success) {
187 : XL_VKAPI_LOG("[", frameIdx, "] vkQueueSubmit [complete]",
188 : " [", platform::clock(core::ClockType::Monotonic) - t, "]");
189 : }, nullptr, "DeviceQueue::submit");
190 : #endif
191 :
192 25305 : _device->makeApiCall([&, this] (const DeviceTable &table, VkDevice device) {
193 25305 : switch (idle) {
194 0 : case IdleMode::Queue:
195 0 : table.vkQueueWaitIdle(_queue);
196 0 : break;
197 0 : case IdleMode::Device:
198 0 : table.vkDeviceWaitIdle(_device->getDevice());
199 0 : break;
200 25305 : case IdleMode::None:
201 25305 : break;
202 : }
203 : #if XL_VKAPI_DEBUG
204 : auto t = platform::clock(core::ClockType::Monotonic);
205 : _result = table.vkQueueSubmit(_queue, 1, &submitInfo, fence.getFence());
206 : XL_VKAPI_LOG("[", _frameIdx, "] vkQueueSubmit: ", _result, " ", (void *)_queue,
207 : " [", platform::clock(core::ClockType::Monotonic) - t, "]");
208 : #else
209 25305 : _result = table.vkQueueSubmit(_queue, 1, &submitInfo, fence.getFence());
210 : #endif
211 25305 : });
212 :
213 25305 : if (_result == VK_SUCCESS) {
214 25305 : fence.setArmed(*this);
215 25305 : return true;
216 : }
217 0 : return false;
218 25305 : }
219 :
220 21 : void DeviceQueue::waitIdle() {
221 21 : _device->makeApiCall([&, this] (const DeviceTable &table, VkDevice device) {
222 21 : table.vkQueueWaitIdle(_queue);
223 21 : });
224 21 : }
225 :
226 0 : uint32_t DeviceQueue::getActiveFencesCount() {
227 0 : return _nfences.load();
228 : }
229 :
230 197901 : void DeviceQueue::retainFence(const Fence &fence) {
231 197901 : ++ _nfences;
232 197902 : }
233 :
234 197903 : void DeviceQueue::releaseFence(const Fence &fence) {
235 197903 : -- _nfences;
236 197903 : }
237 :
238 172577 : void DeviceQueue::setOwner(FrameHandle &frame) {
239 172577 : _frameIdx = frame.getOrder();
240 172577 : }
241 :
242 207399 : void DeviceQueue::reset() {
243 207399 : _result = VK_ERROR_UNKNOWN;
244 207399 : _frameIdx = 0;
245 207399 : }
246 :
247 10856 : ImageMemoryBarrier::ImageMemoryBarrier(Image *image, VkAccessFlags src, VkAccessFlags dst,
248 10856 : VkImageLayout old, VkImageLayout _new)
249 10856 : : srcAccessMask(src), dstAccessMask(dst), oldLayout(old), newLayout(_new)
250 21712 : , image(image), subresourceRange(VkImageSubresourceRange{
251 10856 : image->getAspectMask(), 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS
252 10856 : }) { }
253 :
254 0 : ImageMemoryBarrier::ImageMemoryBarrier(Image *image, VkAccessFlags src, VkAccessFlags dst,
255 0 : VkImageLayout old, VkImageLayout _new, VkImageSubresourceRange range)
256 0 : : srcAccessMask(src), dstAccessMask(dst), oldLayout(old), newLayout(_new)
257 0 : , image(image), subresourceRange(range) { }
258 :
259 10625 : ImageMemoryBarrier::ImageMemoryBarrier(Image *image, VkAccessFlags src, VkAccessFlags dst,
260 10625 : VkImageLayout old, VkImageLayout _new, QueueFamilyTransfer transfer)
261 10625 : : srcAccessMask(src), dstAccessMask(dst), oldLayout(old), newLayout(_new), familyTransfer(transfer)
262 21250 : , image(image), subresourceRange(VkImageSubresourceRange{
263 10625 : image->getAspectMask(), 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS
264 10625 : }) { }
265 :
266 0 : ImageMemoryBarrier::ImageMemoryBarrier(Image *image, VkAccessFlags src, VkAccessFlags dst,
267 0 : VkImageLayout old, VkImageLayout _new, QueueFamilyTransfer transfer, VkImageSubresourceRange range)
268 0 : : srcAccessMask(src), dstAccessMask(dst), oldLayout(old), newLayout(_new), familyTransfer(transfer)
269 0 : , image(image), subresourceRange(range) { }
270 :
271 63 : ImageMemoryBarrier::ImageMemoryBarrier(const VkImageMemoryBarrier &barrier)
272 63 : : srcAccessMask(barrier.srcAccessMask), dstAccessMask(barrier.dstAccessMask)
273 63 : , oldLayout(barrier.oldLayout), newLayout(barrier.newLayout)
274 63 : , familyTransfer(QueueFamilyTransfer{barrier.srcQueueFamilyIndex, barrier.dstQueueFamilyIndex})
275 63 : , image(nullptr), subresourceRange(barrier.subresourceRange) { }
276 :
277 1624503 : BufferMemoryBarrier::BufferMemoryBarrier(Buffer *buf, VkAccessFlags src, VkAccessFlags dst)
278 1624503 : : srcAccessMask(src), dstAccessMask(dst), buffer(buf) { }
279 :
280 0 : BufferMemoryBarrier::BufferMemoryBarrier(Buffer *buf, VkAccessFlags src, VkAccessFlags dst,
281 0 : QueueFamilyTransfer transfer)
282 0 : : srcAccessMask(src), dstAccessMask(dst), familyTransfer(transfer), buffer(buf) { }
283 :
284 19911 : BufferMemoryBarrier::BufferMemoryBarrier(Buffer *buf, VkAccessFlags src, VkAccessFlags dst,
285 19911 : QueueFamilyTransfer transfer, VkDeviceSize offset, VkDeviceSize size)
286 19911 : : srcAccessMask(src), dstAccessMask(dst), familyTransfer(transfer),
287 19911 : buffer(buf), offset(offset), size(size) { }
288 :
289 420 : BufferMemoryBarrier::BufferMemoryBarrier(const VkBufferMemoryBarrier &barrier)
290 420 : : srcAccessMask(barrier.srcAccessMask), dstAccessMask(barrier.dstAccessMask)
291 420 : , familyTransfer(QueueFamilyTransfer{barrier.srcQueueFamilyIndex, barrier.dstQueueFamilyIndex})
292 420 : , buffer(nullptr), offset(barrier.offset), size(barrier.size) { }
293 :
294 :
295 28569 : DescriptorImageInfo::~DescriptorImageInfo() { }
296 :
297 28463 : DescriptorImageInfo::DescriptorImageInfo(const PipelineDescriptor *desc, uint32_t index, bool external)
298 28463 : : DescriptorInfo(desc, index, external) { }
299 :
300 1581483 : DescriptorBufferInfo::~DescriptorBufferInfo() { }
301 :
302 1581483 : DescriptorBufferInfo::DescriptorBufferInfo(const PipelineDescriptor *desc, uint32_t index, bool external)
303 1581483 : : DescriptorInfo(desc, index, external) { }
304 :
305 0 : DescriptorBufferViewInfo::~DescriptorBufferViewInfo() { }
306 :
307 0 : DescriptorBufferViewInfo::DescriptorBufferViewInfo(const PipelineDescriptor *desc, uint32_t index, bool external)
308 0 : : DescriptorInfo(desc, index, external) { }
309 :
310 395806 : CommandBuffer::~CommandBuffer() {
311 197903 : invalidate();
312 395806 : }
313 :
314 197899 : bool CommandBuffer::init(const CommandPool *pool, const DeviceTable *table, VkCommandBuffer buffer) {
315 197899 : _pool = pool;
316 197899 : _table = table;
317 197899 : _buffer = buffer;
318 197899 : return true;
319 : }
320 :
321 197903 : void CommandBuffer::invalidate() {
322 197903 : _buffer = VK_NULL_HANDLE;
323 197903 : }
324 :
325 7378 : void CommandBuffer::cmdPipelineBarrier(VkPipelineStageFlags srcFlags, VkPipelineStageFlags dstFlags, VkDependencyFlags deps,
326 : SpanView<ImageMemoryBarrier> imageBarriers) {
327 7378 : Vector<VkImageMemoryBarrier> images; images.reserve(imageBarriers.size());
328 14798 : for (auto &it : imageBarriers) {
329 7420 : images.emplace_back(VkImageMemoryBarrier{
330 : VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, nullptr,
331 7420 : it.srcAccessMask, it.dstAccessMask,
332 7420 : it.oldLayout, it.newLayout,
333 7420 : it.familyTransfer.srcQueueFamilyIndex, it.familyTransfer.dstQueueFamilyIndex,
334 7420 : it.image->getImage(), it.subresourceRange
335 : });
336 7420 : bindImage(it.image);
337 : }
338 :
339 7378 : _table->vkCmdPipelineBarrier(_buffer, srcFlags, dstFlags, deps, 0, nullptr, 0, nullptr, images.size(), images.data());
340 7378 : }
341 :
342 1061942 : void CommandBuffer::cmdPipelineBarrier(VkPipelineStageFlags srcFlags, VkPipelineStageFlags dstFlags, VkDependencyFlags deps,
343 : SpanView<BufferMemoryBarrier> bufferBarriers) {
344 1061942 : Vector<VkBufferMemoryBarrier> buffers; buffers.reserve(bufferBarriers.size());
345 3031082 : for (auto &it : bufferBarriers) {
346 0 : buffers.emplace_back(VkBufferMemoryBarrier{
347 : VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, nullptr,
348 1969141 : it.srcAccessMask, it.dstAccessMask,
349 1969141 : it.familyTransfer.srcQueueFamilyIndex, it.familyTransfer.dstQueueFamilyIndex,
350 1969141 : it.buffer->getBuffer(), it.offset, it.size
351 : });
352 1969141 : bindBuffer(it.buffer);
353 : }
354 :
355 1061937 : _table->vkCmdPipelineBarrier(_buffer, srcFlags, dstFlags, deps, 0, nullptr, buffers.size(), buffers.data(), 0, nullptr);
356 1061942 : }
357 :
358 :
359 24644 : void CommandBuffer::cmdPipelineBarrier(VkPipelineStageFlags srcFlags, VkPipelineStageFlags dstFlags, VkDependencyFlags deps,
360 : SpanView<BufferMemoryBarrier> bufferBarriers, SpanView<ImageMemoryBarrier> imageBarriers) {
361 24644 : Vector<VkBufferMemoryBarrier> buffers; buffers.reserve(bufferBarriers.size());
362 65247 : for (auto &it : bufferBarriers) {
363 0 : buffers.emplace_back(VkBufferMemoryBarrier{
364 : VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, nullptr,
365 40603 : it.srcAccessMask, it.dstAccessMask,
366 40603 : it.familyTransfer.srcQueueFamilyIndex, it.familyTransfer.dstQueueFamilyIndex,
367 40603 : it.buffer->getBuffer(), it.offset, it.size
368 : });
369 40603 : bindBuffer(it.buffer);
370 : }
371 :
372 24644 : Vector<VkImageMemoryBarrier> images; images.reserve(imageBarriers.size());
373 49330 : for (auto &it : imageBarriers) {
374 24686 : images.emplace_back(VkImageMemoryBarrier{
375 : VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, nullptr,
376 24686 : it.srcAccessMask, it.dstAccessMask,
377 24686 : it.oldLayout, it.newLayout,
378 24686 : it.familyTransfer.srcQueueFamilyIndex, it.familyTransfer.dstQueueFamilyIndex,
379 24686 : it.image->getImage(), it.subresourceRange
380 : });
381 24686 : bindImage(it.image);
382 : }
383 :
384 24644 : _table->vkCmdPipelineBarrier(_buffer, srcFlags, dstFlags, deps, 0, nullptr,
385 24644 : buffers.size(), buffers.data(), images.size(), images.data());
386 24644 : }
387 :
388 154525 : void CommandBuffer::cmdCopyBuffer(Buffer *src, Buffer *dst) {
389 154525 : VkBufferCopy copy{0, 0, std::min(src->getSize(), dst->getSize())};
390 154525 : cmdCopyBuffer(src, dst, makeSpanView(©, 1));
391 154525 : }
392 :
393 0 : void CommandBuffer::cmdCopyBuffer(Buffer *src, Buffer *dst, VkDeviceSize srcOffset, VkDeviceSize dstOffset, VkDeviceSize size) {
394 0 : VkBufferCopy copy{srcOffset, dstOffset, size};
395 0 : cmdCopyBuffer(src, dst, makeSpanView(©, 1));
396 0 : }
397 :
398 154610 : void CommandBuffer::cmdCopyBuffer(Buffer *src, Buffer *dst, SpanView<VkBufferCopy> copy) {
399 154610 : bindBuffer(src);
400 154610 : bindBuffer(dst);
401 :
402 154610 : _table->vkCmdCopyBuffer(_buffer, src->getBuffer(), dst->getBuffer(), copy.size(), copy.data());
403 154610 : }
404 :
405 21 : void CommandBuffer::cmdCopyImage(Image *src, VkImageLayout srcLayout, Image *dst, VkImageLayout dstLayout, VkFilter filter) {
406 21 : auto sourceExtent = src->getInfo().extent;
407 21 : auto targetExtent = dst->getInfo().extent;
408 :
409 21 : if (sourceExtent == targetExtent) {
410 21 : VkImageCopy copy{
411 42 : VkImageSubresourceLayers{src->getAspectMask(), 0, 0, src->getInfo().arrayLayers.get()},
412 : VkOffset3D{0, 0, 0},
413 21 : VkImageSubresourceLayers{dst->getAspectMask(), 0, 0, dst->getInfo().arrayLayers.get()},
414 : VkOffset3D{0, 0, 0},
415 21 : VkExtent3D{targetExtent.width, targetExtent.height, targetExtent.depth}
416 42 : };
417 :
418 21 : _table->vkCmdCopyImage(_buffer, src->getImage(), VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
419 : dst->getImage(), VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ©);
420 : } else {
421 0 : VkImageBlit blit{
422 0 : VkImageSubresourceLayers{VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, src->getInfo().arrayLayers.get()},
423 0 : { VkOffset3D{0, 0, 0}, VkOffset3D{int32_t(sourceExtent.width), int32_t(sourceExtent.height), int32_t(sourceExtent.depth)} },
424 0 : VkImageSubresourceLayers{VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, dst->getInfo().arrayLayers.get()},
425 0 : { VkOffset3D{0, 0, 0}, VkOffset3D{int32_t(targetExtent.width), int32_t(targetExtent.height), int32_t(targetExtent.depth)} },
426 0 : };
427 :
428 0 : _table->vkCmdBlitImage(_buffer, src->getImage(), VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
429 : dst->getImage(), VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &blit, filter);
430 : }
431 21 : }
432 :
433 0 : void CommandBuffer::cmdCopyImage(Image *src, VkImageLayout srcLayout, Image *dst, VkImageLayout dstLayout, const VkImageCopy ©) {
434 0 : bindImage(src);
435 0 : bindImage(dst);
436 :
437 0 : _table->vkCmdCopyImage(_buffer, src->getImage(), srcLayout, dst->getImage(), dstLayout, 1, ©);
438 0 : }
439 :
440 0 : void CommandBuffer::cmdCopyImage(Image *src, VkImageLayout srcLayout, Image *dst, VkImageLayout dstLayout, SpanView<VkImageCopy> copy) {
441 0 : bindImage(src);
442 0 : bindImage(dst);
443 :
444 0 : _table->vkCmdCopyImage(_buffer, src->getImage(), srcLayout, dst->getImage(), dstLayout, copy.size(), copy.data());
445 0 : }
446 :
447 42 : void CommandBuffer::cmdCopyBufferToImage(Buffer *buf, Image *img, VkImageLayout layout, VkDeviceSize offset) {
448 42 : auto &extent = img->getInfo().extent;
449 42 : VkImageSubresourceLayers copyLayers({ img->getAspectMask(), 0, 0, img->getInfo().arrayLayers.get() });
450 :
451 42 : VkBufferImageCopy copyRegion{offset, 0, 0, copyLayers, VkOffset3D{0, 0, 0},
452 42 : VkExtent3D{extent.width, extent.height, extent.depth}};
453 :
454 42 : cmdCopyBufferToImage(buf, img, layout, makeSpanView(©Region, 1));
455 42 : }
456 :
457 2153 : void CommandBuffer::cmdCopyBufferToImage(Buffer *buf, Image *img, VkImageLayout layout, SpanView<VkBufferImageCopy> copy) {
458 2153 : bindBuffer(buf);
459 2153 : bindImage(img);
460 :
461 2153 : _table->vkCmdCopyBufferToImage(_buffer, buf->getBuffer(), img->getImage(), layout, copy.size(), copy.data());
462 2153 : }
463 :
464 21 : void CommandBuffer::cmdCopyImageToBuffer(Image *img, VkImageLayout layout, Buffer *buf, VkDeviceSize offset) {
465 21 : auto &extent = img->getInfo().extent;
466 21 : VkImageSubresourceLayers copyLayers({ img->getAspectMask(), 0, 0, img->getInfo().arrayLayers.get() });
467 :
468 21 : VkBufferImageCopy copyRegion{offset, 0, 0, copyLayers, VkOffset3D{0, 0, 0},
469 21 : VkExtent3D{extent.width, extent.height, extent.depth}};
470 :
471 21 : cmdCopyImageToBuffer(img, layout, buf, makeSpanView(©Region, 1));
472 21 : }
473 :
474 21 : void CommandBuffer::cmdCopyImageToBuffer(Image *img, VkImageLayout layout, Buffer *buf, SpanView<VkBufferImageCopy> copy) {
475 21 : bindBuffer(buf);
476 21 : bindImage(img);
477 :
478 21 : _table->vkCmdCopyImageToBuffer(_buffer, img->getImage(), layout, buf->getBuffer(), copy.size(), copy.data());
479 21 : }
480 :
481 7273 : void CommandBuffer::cmdClearColorImage(Image *image, VkImageLayout layout, const Color4F &color) {
482 : VkClearColorValue clearColorEmpty;
483 7273 : clearColorEmpty.float32[0] = color.r;
484 7273 : clearColorEmpty.float32[1] = color.g;
485 7273 : clearColorEmpty.float32[2] = color.b;
486 7273 : clearColorEmpty.float32[3] = color.a;
487 :
488 7273 : VkImageSubresourceRange range{ image->getAspectMask(), 0, image->getInfo().mipLevels.get(), 0, image->getInfo().arrayLayers.get() };
489 :
490 7273 : bindImage(image);
491 7273 : _table->vkCmdClearColorImage(_buffer, image->getImage(), layout,
492 : &clearColorEmpty, 1, &range);
493 7273 : }
494 :
495 9517 : void CommandBuffer::cmdBeginRenderPass(RenderPass *pass, Framebuffer *fb, VkSubpassContents subpass, bool alt) {
496 9517 : auto &clearValues = pass->getClearValues();
497 9517 : auto currentExtent = fb->getExtent();
498 :
499 9517 : VkRenderPassBeginInfo renderPassInfo {
500 : VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO, nullptr
501 : };
502 9517 : renderPassInfo.sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO;
503 9517 : renderPassInfo.pNext = nullptr;
504 9517 : renderPassInfo.renderPass = pass->getRenderPass(alt);
505 9517 : renderPassInfo.framebuffer = fb->getFramebuffer();
506 9517 : renderPassInfo.renderArea.offset = { 0, 0 };
507 9517 : renderPassInfo.renderArea.extent = VkExtent2D{currentExtent.width, currentExtent.height};
508 9517 : renderPassInfo.clearValueCount = clearValues.size();
509 9517 : renderPassInfo.pClearValues = clearValues.data();
510 :
511 9517 : bindFramebuffer(fb);
512 9517 : _table->vkCmdBeginRenderPass(_buffer, &renderPassInfo, subpass);
513 :
514 9517 : _currentSubpass = 0;
515 9517 : _withinRenderpass = true;
516 9517 : }
517 :
518 9517 : void CommandBuffer::cmdEndRenderPass() {
519 9517 : _table->vkCmdEndRenderPass(_buffer);
520 :
521 9517 : _withinRenderpass = false;
522 9517 : _currentSubpass = 0;
523 9517 : }
524 :
525 11845 : void CommandBuffer::cmdSetViewport(uint32_t firstViewport, SpanView<VkViewport> viewports) {
526 11845 : _table->vkCmdSetViewport(_buffer, firstViewport, viewports.size(), viewports.data());
527 11845 : }
528 :
529 18842 : void CommandBuffer::cmdSetScissor(uint32_t firstScissor, SpanView<VkRect2D> scissors) {
530 : //log::verbose("CommandBuffer", "cmdSetScissor: ", scissors.front().offset.x, " ", scissors.front().offset.y, " ",
531 : // scissors.front().extent.width, " ", scissors.front().extent.height);
532 18842 : _table->vkCmdSetScissor(_buffer, firstScissor, scissors.size(), scissors.data());
533 18842 : }
534 :
535 21362 : void CommandBuffer::cmdBindPipeline(GraphicPipeline *pipeline) {
536 21362 : _table->vkCmdBindPipeline(_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->getPipeline());
537 21362 : }
538 :
539 1246349 : void CommandBuffer::cmdBindPipeline(ComputePipeline *pipeline) {
540 1246349 : _table->vkCmdBindPipeline(_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline->getPipeline());
541 1246349 : }
542 :
543 9517 : void CommandBuffer::cmdBindIndexBuffer(Buffer *buf, VkDeviceSize offset, VkIndexType indexType) {
544 9517 : _table->vkCmdBindIndexBuffer(_buffer, buf->getBuffer(), offset, indexType);
545 9517 : }
546 :
547 240989 : void CommandBuffer::cmdBindDescriptorSets(RenderPass *pass, uint32_t layoutIndex, uint32_t firstSet) {
548 240989 : auto &sets = pass->getDescriptorSets(layoutIndex);
549 :
550 240989 : Vector<VkDescriptorSet> bindSets; bindSets.reserve(sets.size());
551 481975 : for (auto &it : sets) {
552 240987 : bindSets.emplace_back(it->set);
553 240986 : _descriptorSets.emplace(it);
554 : }
555 :
556 240983 : cmdBindDescriptorSets(pass, layoutIndex, bindSets, firstSet);
557 240994 : }
558 :
559 250504 : void CommandBuffer::cmdBindDescriptorSets(RenderPass *pass, uint32_t layoutIndex, SpanView<VkDescriptorSet> sets, uint32_t firstSet) {
560 250504 : auto bindPoint = (pass->getType() == core::PassType::Compute) ? VK_PIPELINE_BIND_POINT_COMPUTE : VK_PIPELINE_BIND_POINT_GRAPHICS;
561 :
562 250504 : _boundLayoutIndex = layoutIndex;
563 250504 : _boundLayout = pass->getPipelineLayout(layoutIndex);
564 :
565 501007 : _table->vkCmdBindDescriptorSets(_buffer, bindPoint, _boundLayout, firstSet,
566 250503 : sets.size(), sets.data(), 0, nullptr);
567 250511 : }
568 :
569 0 : void CommandBuffer::cmdBindGraphicDescriptorSets(VkPipelineLayout layout, SpanView<VkDescriptorSet> sets, uint32_t firstSet) {
570 0 : _table->vkCmdBindDescriptorSets(_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, layout, firstSet,
571 0 : sets.size(), sets.data(), 0, nullptr);
572 0 : }
573 :
574 0 : void CommandBuffer::cmdBindComputeDescriptorSets(VkPipelineLayout layout, SpanView<VkDescriptorSet> sets, uint32_t firstSet) {
575 0 : _table->vkCmdBindDescriptorSets(_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, layout, firstSet,
576 0 : sets.size(), sets.data(), 0, nullptr);
577 0 : }
578 :
579 0 : void CommandBuffer::cmdDraw(uint32_t vertexCount, uint32_t instanceCount, uint32_t firstVertex, uint32_t firstInstance) {
580 0 : _table->vkCmdDraw(_buffer, vertexCount, instanceCount, firstVertex, firstInstance);
581 0 : }
582 45830 : void CommandBuffer::cmdDrawIndexed(uint32_t indexCount, uint32_t instanceCount, uint32_t firstIndex,
583 : int32_t vertexOffset, uint32_t firstInstance) {
584 45830 : _table->vkCmdDrawIndexed(_buffer, indexCount, instanceCount, firstIndex, vertexOffset, firstInstance);
585 45830 : }
586 :
587 43502 : void CommandBuffer::cmdPushConstants(VkPipelineLayout layout, VkShaderStageFlags stageFlags, uint32_t offset, BytesView data) {
588 43502 : _table->vkCmdPushConstants(_buffer, layout, stageFlags, offset, data.size(), data.data());
589 43502 : }
590 :
591 909528 : void CommandBuffer::cmdPushConstants(VkShaderStageFlags stageFlags, uint32_t offset, BytesView data) {
592 909528 : XLASSERT(_boundLayout, "cmdPushConstants without bound layout");
593 909528 : _table->vkCmdPushConstants(_buffer, _boundLayout, stageFlags, offset, data.size(), data.data());
594 909528 : }
595 :
596 2370 : void CommandBuffer::cmdFillBuffer(Buffer *buffer, uint32_t data) {
597 2370 : cmdFillBuffer(buffer, 0, VK_WHOLE_SIZE, data);
598 2370 : }
599 :
600 2370 : void CommandBuffer::cmdFillBuffer(Buffer *buffer, VkDeviceSize dstOffset, VkDeviceSize size, uint32_t data) {
601 2370 : bindBuffer(buffer);
602 2370 : _table->vkCmdFillBuffer(_buffer, buffer->getBuffer(), dstOffset, size, data);
603 2370 : }
604 :
605 1246349 : void CommandBuffer::cmdDispatch(uint32_t groupCountX, uint32_t groupCountY, uint32_t groupCountZ) {
606 1246349 : _table->vkCmdDispatch(_buffer, groupCountX, groupCountY, groupCountZ);
607 1246349 : }
608 :
609 529200 : void CommandBuffer::cmdDispatchPipeline(ComputePipeline *pipeline, uint32_t countX, uint32_t countY, uint32_t countZ) {
610 529200 : cmdBindPipeline(pipeline);
611 529200 : cmdDispatch((countX - 1) / pipeline->getLocalX() + 1,
612 529200 : (countY - 1) / pipeline->getLocalY() + 1,
613 529200 : (countZ - 1) / pipeline->getLocalZ() + 1);
614 529200 : }
615 :
616 59917 : uint32_t CommandBuffer::cmdNextSubpass() {
617 59917 : if (_withinRenderpass) {
618 9517 : _table->vkCmdNextSubpass(_buffer, VK_SUBPASS_CONTENTS_INLINE);
619 9517 : ++ _currentSubpass;
620 9517 : return _currentSubpass;
621 : }
622 50400 : return 0;
623 : }
624 :
625 42 : void CommandBuffer::writeImageTransfer(uint32_t sourceFamily, uint32_t targetFamily, const Rc<Buffer> &buffer, const Rc<Image> &image) {
626 : ImageMemoryBarrier inImageBarrier(image, 0, VK_ACCESS_TRANSFER_WRITE_BIT,
627 42 : VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL);
628 :
629 42 : cmdPipelineBarrier(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, makeSpanView(&inImageBarrier, 1));
630 :
631 42 : auto sourceFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
632 42 : auto targetFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
633 :
634 42 : if (image->getInfo().type != core::PassType::Generic && sourceFamily != VK_QUEUE_FAMILY_IGNORED && targetFamily != VK_QUEUE_FAMILY_IGNORED) {
635 42 : if (sourceFamily != targetFamily) {
636 42 : sourceFamilyIndex = sourceFamily;
637 42 : targetFamilyIndex = targetFamily;
638 : }
639 : }
640 :
641 42 : cmdCopyBufferToImage(buffer, image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 0);
642 :
643 : ImageMemoryBarrier outImageBarrier(image, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT,
644 : VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
645 42 : QueueFamilyTransfer{sourceFamilyIndex, targetFamilyIndex});
646 :
647 42 : cmdPipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, makeSpanView(&outImageBarrier, 1));
648 :
649 42 : if (targetFamilyIndex != VK_QUEUE_FAMILY_IGNORED) {
650 42 : image->setPendingBarrier(outImageBarrier);
651 : }
652 42 : }
653 :
654 2323508 : void CommandBuffer::bindBuffer(core::BufferObject *buffer) {
655 2323508 : core::CommandBuffer::bindBuffer(buffer);
656 2323508 : if (auto pool = static_cast<Buffer *>(buffer)->getMemory()->getPool()) {
657 1481913 : _memPool.emplace(pool);
658 : }
659 2323508 : }
660 :
661 882 : CommandPool::~CommandPool() {
662 441 : if (_commandPool) {
663 0 : log::error("VK-Error", "CommandPool was not destroyed");
664 : }
665 882 : }
666 :
667 441 : bool CommandPool::init(Device &dev, uint32_t familyIdx, QueueOperations c, bool transient) {
668 441 : _familyIdx = familyIdx;
669 441 : _class = c;
670 441 : VkCommandPoolCreateInfo poolInfo{};
671 441 : poolInfo.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
672 441 : poolInfo.pNext = nullptr;
673 441 : poolInfo.queueFamilyIndex = familyIdx;
674 441 : poolInfo.flags = 0; // transient ? VK_COMMAND_POOL_CREATE_TRANSIENT_BIT : 0;
675 :
676 441 : return dev.getTable()->vkCreateCommandPool(dev.getDevice(), &poolInfo, nullptr, &_commandPool) == VK_SUCCESS;
677 : }
678 :
679 441 : void CommandPool::invalidate(Device &dev) {
680 441 : if (_commandPool) {
681 441 : dev.getTable()->vkDestroyCommandPool(dev.getDevice(), _commandPool, nullptr);
682 441 : _commandPool = VK_NULL_HANDLE;
683 : } else {
684 0 : log::error("VK-Error", "CommandPool is not defined");
685 : }
686 441 : }
687 :
688 197903 : const CommandBuffer *CommandPool::recordBuffer(Device &dev, const Callback<bool(CommandBuffer &)> &cb,
689 : VkCommandBufferUsageFlagBits flags, Level level) {
690 197903 : if (!_commandPool) {
691 0 : return nullptr;
692 : }
693 :
694 197903 : VkCommandBufferAllocateInfo allocInfo{};
695 197903 : allocInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
696 197903 : allocInfo.commandPool = _commandPool;
697 197903 : allocInfo.level = VkCommandBufferLevel(level);
698 197903 : allocInfo.commandBufferCount = 1;
699 :
700 : VkCommandBuffer buf;
701 197903 : if (dev.getTable()->vkAllocateCommandBuffers(dev.getDevice(), &allocInfo, &buf) != VK_SUCCESS) {
702 0 : return nullptr;
703 : }
704 :
705 :
706 197903 : VkCommandBufferBeginInfo beginInfo { };
707 197903 : beginInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
708 197903 : beginInfo.pNext = nullptr;
709 197903 : beginInfo.flags = flags;
710 197903 : beginInfo.pInheritanceInfo = nullptr;
711 :
712 197903 : if (dev.getTable()->vkBeginCommandBuffer(buf, &beginInfo) != VK_SUCCESS) {
713 0 : dev.getTable()->vkFreeCommandBuffers(dev.getDevice(), _commandPool, 1, &buf);
714 0 : return nullptr;
715 : }
716 :
717 197903 : auto b = Rc<CommandBuffer>::create(this, dev.getTable(), buf);
718 197899 : if (!b) {
719 0 : dev.getTable()->vkEndCommandBuffer(buf);
720 0 : dev.getTable()->vkFreeCommandBuffers(dev.getDevice(), _commandPool, 1, &buf);
721 0 : return nullptr;
722 : }
723 :
724 197899 : auto result = cb(*b);
725 :
726 197903 : dev.getTable()->vkEndCommandBuffer(buf);
727 :
728 197903 : if (!result) {
729 0 : dev.getTable()->vkFreeCommandBuffers(dev.getDevice(), _commandPool, 1, &buf);
730 0 : return nullptr;
731 : }
732 :
733 197903 : return _buffers.emplace_back(move(b)).get();
734 197902 : }
735 :
736 0 : void CommandPool::freeDefaultBuffers(Device &dev, Vector<VkCommandBuffer> &vec) {
737 0 : if (_commandPool) {
738 0 : dev.getTable()->vkFreeCommandBuffers(dev.getDevice(), _commandPool, static_cast<uint32_t>(vec.size()), vec.data());
739 : }
740 0 : vec.clear();
741 0 : }
742 :
743 395781 : void CommandPool::reset(Device &dev, bool release) {
744 395781 : if (_commandPool) {
745 : //dev.getTable()->vkResetCommandPool(dev.getDevice(), _commandPool, release ? VK_COMMAND_POOL_RESET_RELEASE_RESOURCES_BIT : 0);
746 :
747 395781 : Vector<VkCommandBuffer> buffersToFree;
748 593683 : for (auto &it : _buffers) {
749 197903 : if (it) {
750 197903 : buffersToFree.emplace_back(it->getBuffer());
751 : }
752 : }
753 395779 : if (buffersToFree.size() > 0) {
754 197903 : dev.getTable()->vkFreeCommandBuffers(dev.getDevice(), _commandPool, static_cast<uint32_t>(buffersToFree.size()), buffersToFree.data());
755 : }
756 395784 : dev.getTable()->vkDestroyCommandPool(dev.getDevice(), _commandPool, nullptr);
757 :
758 395784 : VkCommandPoolCreateInfo poolInfo{};
759 395784 : poolInfo.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
760 395784 : poolInfo.pNext = nullptr;
761 395784 : poolInfo.queueFamilyIndex = _familyIdx;
762 395784 : poolInfo.flags = 0; // transient ? VK_COMMAND_POOL_CREATE_TRANSIENT_BIT : 0;
763 :
764 395784 : dev.getTable()->vkCreateCommandPool(dev.getDevice(), &poolInfo, nullptr, &_commandPool);
765 :
766 395785 : _buffers.clear();
767 395779 : _autorelease.clear();
768 395780 : }
769 395782 : }
770 :
771 656158 : void CommandPool::autorelease(Rc<Ref> &&ref) {
772 656158 : _autorelease.emplace_back(move(ref));
773 656126 : }
774 :
775 : }
|
