Line data Source code
1 : /**
2 : Copyright (c) 2021-2022 Roman Katuntsev <sbkarr@stappler.org>
3 : Copyright (c) 2023 Stappler LLC <admin@stappler.dev>
4 :
5 : Permission is hereby granted, free of charge, to any person obtaining a copy
6 : of this software and associated documentation files (the "Software"), to deal
7 : in the Software without restriction, including without limitation the rights
8 : to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 : copies of the Software, and to permit persons to whom the Software is
10 : furnished to do so, subject to the following conditions:
11 :
12 : The above copyright notice and this permission notice shall be included in
13 : all copies or substantial portions of the Software.
14 :
15 : THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 : IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 : FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 : AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 : LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 : OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 : THE SOFTWARE.
22 : **/
23 :
24 : #ifndef XENOLITH_BACKEND_VK_XLVKALLOCATOR_H_
25 : #define XENOLITH_BACKEND_VK_XLVKALLOCATOR_H_
26 :
27 : #include "XLVkInfo.h"
28 :
29 : namespace STAPPLER_VERSIONIZED stappler::xenolith::vk {
30 :
31 : class Device;
32 : class DeviceMemory;
33 : class Image;
34 : class Buffer;
35 :
36 : enum class AllocationUsage {
37 : DeviceLocal, // device local only
38 : DeviceLocalHostVisible, // device local, visible directly on host
39 : HostTransitionSource, // host-local, used as source for transfer to GPU device (so, non-cached, coherent preferable)
40 : HostTransitionDestination, // host-local, used as destination for transfer from GPU Device (cached, non-coherent)
41 : DeviceLocalLazilyAllocated, // memory for transient images
42 : };
43 :
44 : enum class AllocationType {
45 : Unknown,
46 : Linear,
47 : Optimal,
48 : };
49 :
50 : struct MemoryRequirements {
51 : VkMemoryRequirements requirements;
52 : VkDeviceSize targetOffset;
53 : bool prefersDedicated = false;
54 : bool requiresDedicated = false;
55 : };
56 :
57 : class Allocator : public Ref {
58 : public:
59 : static constexpr uint64_t PageSize = 8_MiB;
60 : static constexpr uint64_t MaxIndex = 20; // Largest block
61 : static constexpr uint64_t PreservePages = 20;
62 :
63 : enum MemHeapType {
64 : HostLocal,
65 : DeviceLocal,
66 : DeviceLocalHostVisible,
67 : };
68 :
69 : struct MemHeap;
70 :
71 : // slice of device memory
72 : struct MemNode {
73 : uint64_t index = 0; // size in pages
74 : VkDeviceMemory mem = VK_NULL_HANDLE; // device mem block
75 : VkDeviceSize size = 0; // size in bytes
76 : VkDeviceSize offset = 0; // current usage offset
77 : AllocationType lastAllocation = AllocationType::Unknown; // last allocation type (for bufferImageGranularity)
78 : void *ptr = nullptr;
79 : Mutex *mappingProtection = nullptr;
80 :
81 : explicit operator bool () const { return mem != VK_NULL_HANDLE; }
82 :
83 : size_t getFreeSpace() const { return size - offset; }
84 : };
85 :
86 : // Memory block, allocated from node as suballocation
87 : struct MemBlock {
88 : VkDeviceMemory mem = VK_NULL_HANDLE; // device mem block
89 : VkDeviceSize offset = 0; // offset in node
90 : VkDeviceSize size = 0; // reserved size after offset
91 : uint32_t type = 0; // memory type index
92 : void *ptr = nullptr;
93 : Mutex *mappingProtection = nullptr;
94 :
95 695827 : explicit operator bool () const { return mem != VK_NULL_HANDLE; }
96 : };
97 :
98 : struct MemType {
99 : uint32_t idx;
100 : VkMemoryType type;
101 : uint64_t min = 2; // in PageSize
102 : uint64_t last = 0; // largest used index into free
103 : uint64_t max = PreservePages; // Pages to preserve, 0 - do not preserve
104 : uint64_t current = PreservePages; // current allocated size in BOUNDARY_SIZE
105 : std::array<Vector<MemNode>, MaxIndex> buf;
106 :
107 1336094 : bool isDeviceLocal() const { return (type.propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0; }
108 2711769 : bool isHostVisible() const { return (type.propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0; }
109 1911029 : bool isHostCoherent() const { return (type.propertyFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0; }
110 1320555 : bool isHostCached() const { return (type.propertyFlags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0; }
111 42 : bool isLazilyAllocated() const { return (type.propertyFlags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0; }
112 : bool isProtected() const { return (type.propertyFlags & VK_MEMORY_PROPERTY_PROTECTED_BIT) != 0; }
113 : };
114 :
115 : struct MemHeap {
116 : uint32_t idx;
117 : VkMemoryHeap heap;
118 : Vector<MemType> types;
119 : MemHeapType type = HostLocal;
120 : VkDeviceSize budget = 0;
121 : VkDeviceSize usage = 0;
122 : VkDeviceSize currentUsage = 0;
123 : };
124 :
125 : virtual ~Allocator();
126 :
127 : bool init(Device &dev, VkPhysicalDevice device, const DeviceInfo::Features &features, const DeviceInfo::Properties &props);
128 : void invalidate(Device &dev);
129 :
130 : void update();
131 :
132 : uint32_t getInitialTypeMask() const;
133 : const Vector<MemHeap> &getMemHeaps() const { return _memHeaps; }
134 709226 : Device *getDevice() const { return _device; }
135 :
136 : bool hasBudgetFeature() const { return _hasBudget; }
137 701077 : bool hasMemReq2Feature() const { return _hasMemReq2; }
138 702543 : bool hasDedicatedFeature() const { return _hasDedicated; }
139 1171 : VkDeviceSize getBufferImageGranularity() const { return _bufferImageGranularity; }
140 63 : VkDeviceSize getNonCoherentAtomSize() const { return _nonCoherentAtomSize; }
141 :
142 : Mutex &getMutex() const { return _mutex; }
143 :
144 : const MemType *getType(uint32_t) const;
145 :
146 : MemType * findMemoryType(uint32_t typeFilter, AllocationUsage) const;
147 :
148 : MemoryRequirements getBufferMemoryRequirements(VkBuffer target);
149 : MemoryRequirements getImageMemoryRequirements(VkImage target);
150 :
151 : Rc<Buffer> spawnPersistent(AllocationUsage, const BufferInfo &, BytesView = BytesView());
152 : Rc<Image> spawnPersistent(AllocationUsage, const ImageInfoData &, bool preinitialized, uint64_t forceId = 0);
153 :
154 : Rc<Buffer> preallocate(const BufferInfo &, BytesView = BytesView());
155 : Rc<Image> preallocate(const ImageInfoData &, bool preinitialized, uint64_t forceId = 0);
156 :
157 : Rc<DeviceMemory> emplaceObjects(AllocationUsage usage, SpanView<Rc<Image>>, SpanView<Rc<Buffer>>);
158 :
159 : protected:
160 : friend class DeviceMemoryPool;
161 :
162 : void lock();
163 : void unlock();
164 :
165 : MemNode alloc(MemType *, uint64_t, bool persistent = false);
166 : void free(MemType *, SpanView<MemNode>);
167 :
168 : bool allocateDedicated(AllocationUsage usage, Buffer *);
169 : bool allocateDedicated(AllocationUsage usage, Image *);
170 :
171 : mutable Mutex _mutex;
172 : VkPhysicalDevice _physicalDevice = VK_NULL_HANDLE;
173 : Device *_device = nullptr;
174 : VkPhysicalDeviceMemoryBudgetPropertiesEXT _memBudget = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT };
175 : VkPhysicalDeviceMemoryProperties2KHR _memProperties = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2_KHR };
176 : Vector<MemHeap> _memHeaps;
177 : Vector<const MemType *> _memTypes;
178 :
179 : VkDeviceSize _bufferImageGranularity = 1;
180 : VkDeviceSize _nonCoherentAtomSize = 1;
181 : bool _hasBudget = false;
182 : bool _hasMemReq2 = false;
183 : bool _hasDedicated = false;
184 : };
185 :
186 : class DeviceMemoryPool : public Ref {
187 : public:
188 : struct MemData {
189 : Allocator::MemType *type = nullptr;
190 : Vector<Allocator::MemNode> mem;
191 : Vector<Allocator::MemBlock> freed;
192 : };
193 :
194 : virtual ~DeviceMemoryPool();
195 :
196 : bool init(const Rc<Allocator> &, bool persistentMapping = false);
197 :
198 : Rc<Buffer> spawn(AllocationUsage type, const BufferInfo &);
199 : Rc<Image> spawn(AllocationUsage type, const ImageInfoData &);
200 :
201 : Rc<Buffer> spawnPersistent(AllocationUsage, const BufferInfo &);
202 :
203 : Device *getDevice() const;
204 695848 : const Rc<Allocator> &getAllocator() const { return _allocator; }
205 :
206 : Mutex &getMutex() { return _mutex; }
207 :
208 : Allocator::MemBlock alloc(MemData *, VkDeviceSize size, VkDeviceSize alignment, AllocationType allocType, AllocationUsage type);
209 : void free(Allocator::MemBlock &&);
210 :
211 : protected:
212 : void clear(MemData *);
213 :
214 : Mutex _mutex;
215 : bool _persistentMapping = false;
216 : Rc<Allocator> _allocator;
217 : Map<int64_t, MemData> _heaps;
218 : Map<VkDeviceMemory, Mutex *> _mappingProtection;
219 : std::forward_list<Rc<Buffer>> _buffers;
220 : std::forward_list<Rc<Image>> _images;
221 : };
222 :
223 : }
224 :
225 : namespace std {
226 :
227 0 : inline std::ostream &operator<<(std::ostream &stream, STAPPLER_VERSIONIZED_NAMESPACE::xenolith::vk::AllocationUsage usage) {
228 0 : switch (usage) {
229 0 : case STAPPLER_VERSIONIZED_NAMESPACE::xenolith::vk::AllocationUsage::DeviceLocal: stream << "DeviceLocal"; break;
230 0 : case STAPPLER_VERSIONIZED_NAMESPACE::xenolith::vk::AllocationUsage::DeviceLocalHostVisible: stream << "DeviceLocalHostVisible"; break;
231 0 : case STAPPLER_VERSIONIZED_NAMESPACE::xenolith::vk::AllocationUsage::DeviceLocalLazilyAllocated: stream << "DeviceLocalLazilyAllocated"; break;
232 0 : case STAPPLER_VERSIONIZED_NAMESPACE::xenolith::vk::AllocationUsage::HostTransitionSource: stream << "HostTransitionSource"; break;
233 0 : case STAPPLER_VERSIONIZED_NAMESPACE::xenolith::vk::AllocationUsage::HostTransitionDestination: stream << "HostTransitionDestination"; break;
234 : }
235 0 : return stream;
236 : }
237 :
238 : }
239 :
240 : #endif /* XENOLITH_BACKEND_VK_XLVKALLOCATOR_H_ */
|
