1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247
#![deny(unsafe_op_in_unsafe_fn)]
use crate::alloc::{GlobalAlloc, Layout, System};
use crate::ffi::c_void;
use crate::ptr;
use crate::sync::atomic::{AtomicPtr, Ordering};
use crate::sys::c;
use crate::sys::common::alloc::{realloc_fallback, MIN_ALIGN};
#[cfg(test)]
mod tests;
// Heap memory management on Windows is done by using the system Heap API (heapapi.h)
// See https://docs.microsoft.com/windows/win32/api/heapapi/
// Flag to indicate that the memory returned by `HeapAlloc` should be zeroed.
const HEAP_ZERO_MEMORY: c::DWORD = 0x00000008;
#[link(name = "kernel32")]
extern "system" {
// Get a handle to the default heap of the current process, or null if the operation fails.
//
// SAFETY: Successful calls to this function within the same process are assumed to
// always return the same handle, which remains valid for the entire lifetime of the process.
//
// See https://docs.microsoft.com/windows/win32/api/heapapi/nf-heapapi-getprocessheap
fn GetProcessHeap() -> c::HANDLE;
// Allocate a block of `dwBytes` bytes of memory from a given heap `hHeap`.
// The allocated memory may be uninitialized, or zeroed if `dwFlags` is
// set to `HEAP_ZERO_MEMORY`.
//
// Returns a pointer to the newly-allocated memory or null if the operation fails.
// The returned pointer will be aligned to at least `MIN_ALIGN`.
//
// SAFETY:
// - `hHeap` must be a non-null handle returned by `GetProcessHeap`.
// - `dwFlags` must be set to either zero or `HEAP_ZERO_MEMORY`.
//
// Note that `dwBytes` is allowed to be zero, contrary to some other allocators.
//
// See https://docs.microsoft.com/windows/win32/api/heapapi/nf-heapapi-heapalloc
fn HeapAlloc(hHeap: c::HANDLE, dwFlags: c::DWORD, dwBytes: c::SIZE_T) -> c::LPVOID;
// Reallocate a block of memory behind a given pointer `lpMem` from a given heap `hHeap`,
// to a block of at least `dwBytes` bytes, either shrinking the block in place,
// or allocating at a new location, copying memory, and freeing the original location.
//
// Returns a pointer to the reallocated memory or null if the operation fails.
// The returned pointer will be aligned to at least `MIN_ALIGN`.
// If the operation fails the given block will never have been freed.
//
// SAFETY:
// - `hHeap` must be a non-null handle returned by `GetProcessHeap`.
// - `dwFlags` must be set to zero.
// - `lpMem` must be a non-null pointer to an allocated block returned by `HeapAlloc` or
// `HeapReAlloc`, that has not already been freed.
// If the block was successfully reallocated at a new location, pointers pointing to
// the freed memory, such as `lpMem`, must not be dereferenced ever again.
//
// Note that `dwBytes` is allowed to be zero, contrary to some other allocators.
//
// See https://docs.microsoft.com/windows/win32/api/heapapi/nf-heapapi-heaprealloc
fn HeapReAlloc(
hHeap: c::HANDLE,
dwFlags: c::DWORD,
lpMem: c::LPVOID,
dwBytes: c::SIZE_T,
) -> c::LPVOID;
// Free a block of memory behind a given pointer `lpMem` from a given heap `hHeap`.
// Returns a nonzero value if the operation is successful, and zero if the operation fails.
//
// SAFETY:
// - `hHeap` must be a non-null handle returned by `GetProcessHeap`.
// - `dwFlags` must be set to zero.
// - `lpMem` must be a pointer to an allocated block returned by `HeapAlloc` or `HeapReAlloc`,
// that has not already been freed.
// If the block was successfully freed, pointers pointing to the freed memory, such as `lpMem`,
// must not be dereferenced ever again.
//
// Note that `lpMem` is allowed to be null, which will not cause the operation to fail.
//
// See https://docs.microsoft.com/windows/win32/api/heapapi/nf-heapapi-heapfree
fn HeapFree(hHeap: c::HANDLE, dwFlags: c::DWORD, lpMem: c::LPVOID) -> c::BOOL;
}
// Cached handle to the default heap of the current process.
// Either a non-null handle returned by `GetProcessHeap`, or null when not yet initialized or `GetProcessHeap` failed.
static HEAP: AtomicPtr<c_void> = AtomicPtr::new(ptr::null_mut());
// Get a handle to the default heap of the current process, or null if the operation fails.
// If this operation is successful, `HEAP` will be successfully initialized and contain
// a non-null handle returned by `GetProcessHeap`.
#[inline]
fn init_or_get_process_heap() -> c::HANDLE {
let heap = HEAP.load(Ordering::Relaxed);
if heap.is_null() {
// `HEAP` has not yet been successfully initialized
let heap = unsafe { GetProcessHeap() };
if !heap.is_null() {
// SAFETY: No locking is needed because within the same process,
// successful calls to `GetProcessHeap` will always return the same value, even on different threads.
HEAP.store(heap, Ordering::Release);
// SAFETY: `HEAP` contains a non-null handle returned by `GetProcessHeap`
heap
} else {
// Could not get the current process heap.
ptr::null_mut()
}
} else {
// SAFETY: `HEAP` contains a non-null handle returned by `GetProcessHeap`
heap
}
}
// Get a non-null handle to the default heap of the current process.
// SAFETY: `HEAP` must have been successfully initialized.
#[inline]
unsafe fn get_process_heap() -> c::HANDLE {
HEAP.load(Ordering::Acquire)
}
// Header containing a pointer to the start of an allocated block.
// SAFETY: Size and alignment must be <= `MIN_ALIGN`.
#[repr(C)]
struct Header(*mut u8);
// Allocate a block of optionally zeroed memory for a given `layout`.
// SAFETY: Returns a pointer satisfying the guarantees of `System` about allocated pointers,
// or null if the operation fails. If this returns non-null `HEAP` will have been successfully
// initialized.
#[inline]
unsafe fn allocate(layout: Layout, zeroed: bool) -> *mut u8 {
let heap = init_or_get_process_heap();
if heap.is_null() {
// Allocation has failed, could not get the current process heap.
return ptr::null_mut();
}
// Allocated memory will be either zeroed or uninitialized.
let flags = if zeroed { HEAP_ZERO_MEMORY } else { 0 };
if layout.align() <= MIN_ALIGN {
// SAFETY: `heap` is a non-null handle returned by `GetProcessHeap`.
// The returned pointer points to the start of an allocated block.
unsafe { HeapAlloc(heap, flags, layout.size()) as *mut u8 }
} else {
// Allocate extra padding in order to be able to satisfy the alignment.
let total = layout.align() + layout.size();
// SAFETY: `heap` is a non-null handle returned by `GetProcessHeap`.
let ptr = unsafe { HeapAlloc(heap, flags, total) as *mut u8 };
if ptr.is_null() {
// Allocation has failed.
return ptr::null_mut();
}
// Create a correctly aligned pointer offset from the start of the allocated block,
// and write a header before it.
let offset = layout.align() - (ptr.addr() & (layout.align() - 1));
// SAFETY: `MIN_ALIGN` <= `offset` <= `layout.align()` and the size of the allocated
// block is `layout.align() + layout.size()`. `aligned` will thus be a correctly aligned
// pointer inside the allocated block with at least `layout.size()` bytes after it and at
// least `MIN_ALIGN` bytes of padding before it.
let aligned = unsafe { ptr.add(offset) };
// SAFETY: Because the size and alignment of a header is <= `MIN_ALIGN` and `aligned`
// is aligned to at least `MIN_ALIGN` and has at least `MIN_ALIGN` bytes of padding before
// it, it is safe to write a header directly before it.
unsafe { ptr::write((aligned as *mut Header).sub(1), Header(ptr)) };
// SAFETY: The returned pointer does not point to the to the start of an allocated block,
// but there is a header readable directly before it containing the location of the start
// of the block.
aligned
}
}
// All pointers returned by this allocator have, in addition to the guarantees of `GlobalAlloc`, the
// following properties:
//
// If the pointer was allocated or reallocated with a `layout` specifying an alignment <= `MIN_ALIGN`
// the pointer will be aligned to at least `MIN_ALIGN` and point to the start of the allocated block.
//
// If the pointer was allocated or reallocated with a `layout` specifying an alignment > `MIN_ALIGN`
// the pointer will be aligned to the specified alignment and not point to the start of the allocated block.
// Instead there will be a header readable directly before the returned pointer, containing the actual
// location of the start of the block.
#[stable(feature = "alloc_system_type", since = "1.28.0")]
unsafe impl GlobalAlloc for System {
#[inline]
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
// SAFETY: Pointers returned by `allocate` satisfy the guarantees of `System`
let zeroed = false;
unsafe { allocate(layout, zeroed) }
}
#[inline]
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
// SAFETY: Pointers returned by `allocate` satisfy the guarantees of `System`
let zeroed = true;
unsafe { allocate(layout, zeroed) }
}
#[inline]
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
let block = {
if layout.align() <= MIN_ALIGN {
ptr
} else {
// The location of the start of the block is stored in the padding before `ptr`.
// SAFETY: Because of the contract of `System`, `ptr` is guaranteed to be non-null
// and have a header readable directly before it.
unsafe { ptr::read((ptr as *mut Header).sub(1)).0 }
}
};
// SAFETY: because `ptr` has been successfully allocated with this allocator,
// `HEAP` must have been successfully initialized.
let heap = unsafe { get_process_heap() };
// SAFETY: `heap` is a non-null handle returned by `GetProcessHeap`,
// `block` is a pointer to the start of an allocated block.
unsafe { HeapFree(heap, 0, block as c::LPVOID) };
}
#[inline]
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
if layout.align() <= MIN_ALIGN {
// SAFETY: because `ptr` has been successfully allocated with this allocator,
// `HEAP` must have been successfully initialized.
let heap = unsafe { get_process_heap() };
// SAFETY: `heap` is a non-null handle returned by `GetProcessHeap`,
// `ptr` is a pointer to the start of an allocated block.
// The returned pointer points to the start of an allocated block.
unsafe { HeapReAlloc(heap, 0, ptr as c::LPVOID, new_size) as *mut u8 }
} else {
// SAFETY: `realloc_fallback` is implemented using `dealloc` and `alloc`, which will
// correctly handle `ptr` and return a pointer satisfying the guarantees of `System`
unsafe { realloc_fallback(self, ptr, layout, new_size) }
}
}
}