[libc++] Implement P1007R3: std::assume_aligned

This supersedes and incoroporates content from both D108906 and D54966,
and also some original content.

Co-Authored-by: Marshall Clow <mclow.lists@gmail.com>
Co-Authored-by: Gonzalo Brito Gadeschi

Differential Revision: https://reviews.llvm.org/D118938

libcxx/docs/FeatureTestMacroTable.rst

@@ -170,7 +170,7 @@ Status
     -------------------------------------------------------------------
     ``__cpp_lib_array_constexpr``                     ``201811L``
     ------------------------------------------------- -----------------
-    ``__cpp_lib_assume_aligned``                      *unimplemented*
+    ``__cpp_lib_assume_aligned``                      ``201811L``
     ------------------------------------------------- -----------------
     ``__cpp_lib_atomic_flag_test``                    ``201907L``
     ------------------------------------------------- -----------------

libcxx/docs/Status/Cxx20Papers.csv

@@ -66,7 +66,7 @@
 "`P0919R3 <https://wg21.link/P0919R3>`__","LWG","Heterogeneous lookup for unordered containers","San Diego","|Complete|","12.0"
 "`P0972R0 <https://wg21.link/P0972R0>`__","LWG","<chrono> ``zero()``\ , ``min()``\ , and ``max()``\  should be noexcept","San Diego","|Complete|","8.0"
 "`P1006R1 <https://wg21.link/P1006R1>`__","LWG","Constexpr in std::pointer_traits","San Diego","|Complete|","8.0"
-"`P1007R3 <https://wg21.link/P1007R3>`__","LWG","``std::assume_aligned``\ ","San Diego","* *",""
+"`P1007R3 <https://wg21.link/P1007R3>`__","LWG","``std::assume_aligned``\ ","San Diego","|Complete|","15.0"
 "`P1020R1 <https://wg21.link/P1020R1>`__","LWG","Smart pointer creation with default initialization","San Diego","* *",""
 "`P1032R1 <https://wg21.link/P1032R1>`__","LWG","Misc constexpr bits","San Diego","|Complete|","13.0"
 "`P1085R2 <https://wg21.link/P1085R2>`__","LWG","Should Span be Regular?","San Diego","|Complete|","8.0"

libcxx/include/CMakeLists.txt

@@ -291,6 +291,7 @@ set(files
   __memory/allocator.h
   __memory/allocator_arg_t.h
   __memory/allocator_traits.h
+  __memory/assume_aligned.h
   __memory/auto_ptr.h
   __memory/compressed_pair.h
   __memory/concepts.h

libcxx/include/__memory/assume_aligned.h

@@ -0,0 +1,46 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_ASSUME_ALIGNED_H
+#define _LIBCPP___MEMORY_ASSUME_ALIGNED_H
+
+#include <__assert>
+#include <__config>
+#include <cstddef>
+#include <cstdint>
+#include <type_traits> // for is_constant_evaluated()
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER > 17
+
+template <size_t _Np, class _Tp>
+[[nodiscard]]
+_LIBCPP_HIDE_FROM_ABI
+constexpr _Tp* assume_aligned(_Tp* __ptr) {
+  static_assert(_Np != 0 && (_Np & (_Np - 1)) == 0,
+    "std::assume_aligned<N>(p) requires N to be a power of two");
+
+  if (is_constant_evaluated()) {
+    return __ptr;
+  } else {
+    _LIBCPP_ASSERT(reinterpret_cast<uintptr_t>(__ptr) % _Np == 0, "Alignment assumption is violated");
+    return static_cast<_Tp*>(__builtin_assume_aligned(__ptr, _Np));
+  }
+}
+
+#endif // _LIBCPP_STD_VER > 17
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MEMORY_ASSUME_ALIGNED_H

libcxx/include/memory

@@ -828,8 +828,12 @@ template <class T> struct hash<shared_ptr<T> >;
 template <class T, class Alloc>
   inline constexpr bool uses_allocator_v = uses_allocator<T, Alloc>::value;
 
+// [ptr.align]
 void* align(size_t alignment, size_t size, void*& ptr, size_t& space);
 
+template<size_t N, class T>
+[[nodiscard]] constexpr T* assume_aligned(T* ptr); // since C++20
+
 }  // std
 
 */
@@ -842,6 +846,7 @@ void* align(size_t alignment, size_t size, void*& ptr, size_t& space);
 #include <__memory/allocator.h>
 #include <__memory/allocator_arg_t.h>
 #include <__memory/allocator_traits.h>
+#include <__memory/assume_aligned.h>
 #include <__memory/compressed_pair.h>
 #include <__memory/concepts.h>
 #include <__memory/construct_at.h>
@@ -1130,7 +1135,6 @@ struct __builtin_new_allocator {
   }
 };
 
-
 _LIBCPP_END_NAMESPACE_STD
 
 #if defined(_LIBCPP_HAS_PARALLEL_ALGORITHMS) && _LIBCPP_STD_VER >= 17

libcxx/include/module.modulemap

@@ -713,6 +713,7 @@ module std [system] {
       module allocator                       { private header "__memory/allocator.h" }
       module allocator_arg_t                 { private header "__memory/allocator_arg_t.h" }
       module allocator_traits                { private header "__memory/allocator_traits.h" }
+      module assume_aligned                  { private header "__memory/assume_aligned.h" }
       module auto_ptr                        { private header "__memory/auto_ptr.h" }
       module compressed_pair                 { private header "__memory/compressed_pair.h" }
       module concepts                        { private header "__memory/concepts.h" }

libcxx/include/version

@@ -288,7 +288,7 @@ __cpp_lib_void_t                                        201411L <type_traits>
 #if _LIBCPP_STD_VER > 17
 # undef  __cpp_lib_array_constexpr
 # define __cpp_lib_array_constexpr                      201811L
-// # define __cpp_lib_assume_aligned                       201811L
+# define __cpp_lib_assume_aligned                       201811L
 # define __cpp_lib_atomic_flag_test                     201907L
 // # define __cpp_lib_atomic_float                         201711L
 # define __cpp_lib_atomic_lock_free_type_aliases        201907L

libcxx/test/libcxx/private_headers.verify.cpp

@@ -322,6 +322,7 @@ END-SCRIPT
 #include <__memory/allocator.h> // expected-error@*:* {{use of private header from outside its module: '__memory/allocator.h'}}
 #include <__memory/allocator_arg_t.h> // expected-error@*:* {{use of private header from outside its module: '__memory/allocator_arg_t.h'}}
 #include <__memory/allocator_traits.h> // expected-error@*:* {{use of private header from outside its module: '__memory/allocator_traits.h'}}
+#include <__memory/assume_aligned.h> // expected-error@*:* {{use of private header from outside its module: '__memory/assume_aligned.h'}}
 #include <__memory/auto_ptr.h> // expected-error@*:* {{use of private header from outside its module: '__memory/auto_ptr.h'}}
 #include <__memory/compressed_pair.h> // expected-error@*:* {{use of private header from outside its module: '__memory/compressed_pair.h'}}
 #include <__memory/concepts.h> // expected-error@*:* {{use of private header from outside its module: '__memory/concepts.h'}}

libcxx/test/libcxx/utilities/memory/ptr.align/assume_aligned.power2.verify.cpp

@@ -0,0 +1,32 @@
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
+
+// #include <memory>
+
+// template<size_t N, class T>
+// [[nodiscard]] constexpr T* assume_aligned(T* ptr);
+
+// This test checks that we static_assert inside std::assume_aligned<N>(p)
+// when N is not a power of two. However, Clang will already emit an error
+// in its own __builtin_assume_aligned, so we ignore that additional error
+// for the purpose of this test. We also ignore the additional warning about
+// remainder by 0 being undefined.
+// ADDITIONAL_COMPILE_FLAGS: -Xclang -verify-ignore-unexpected=error -Xclang -verify-ignore-unexpected=warning
+
+#include <memory>
+
+void f() {
+  int *p = nullptr;
+  (void)std::assume_aligned<0>(p);  // expected-error@*:* {{std::assume_aligned<N>(p) requires N to be a power of two}}
+  (void)std::assume_aligned<3>(p);  // expected-error@*:* {{std::assume_aligned<N>(p) requires N to be a power of two}}
+  (void)std::assume_aligned<5>(p);  // expected-error@*:* {{std::assume_aligned<N>(p) requires N to be a power of two}}
+  (void)std::assume_aligned<33>(p); // expected-error@*:* {{std::assume_aligned<N>(p) requires N to be a power of two}}
+}

libcxx/test/std/language.support/support.limits/support.limits.general/memory.version.compile.pass.cpp

@@ -300,17 +300,11 @@
 #   error "__cpp_lib_allocator_traits_is_always_equal should have the value 201411L in c++20"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_assume_aligned
-#     error "__cpp_lib_assume_aligned should be defined in c++20"
-#   endif
-#   if __cpp_lib_assume_aligned != 201811L
-#     error "__cpp_lib_assume_aligned should have the value 201811L in c++20"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_assume_aligned
-#     error "__cpp_lib_assume_aligned should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_assume_aligned
+#   error "__cpp_lib_assume_aligned should be defined in c++20"
+# endif
+# if __cpp_lib_assume_aligned != 201811L
+#   error "__cpp_lib_assume_aligned should have the value 201811L in c++20"
 # endif
 
 # ifndef __cpp_lib_atomic_value_initialization
@@ -436,17 +430,11 @@
 #   error "__cpp_lib_allocator_traits_is_always_equal should have the value 201411L in c++2b"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_assume_aligned
-#     error "__cpp_lib_assume_aligned should be defined in c++2b"
-#   endif
-#   if __cpp_lib_assume_aligned != 201811L
-#     error "__cpp_lib_assume_aligned should have the value 201811L in c++2b"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_assume_aligned
-#     error "__cpp_lib_assume_aligned should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_assume_aligned
+#   error "__cpp_lib_assume_aligned should be defined in c++2b"
+# endif
+# if __cpp_lib_assume_aligned != 201811L
+#   error "__cpp_lib_assume_aligned should have the value 201811L in c++2b"
 # endif
 
 # ifndef __cpp_lib_atomic_value_initialization

libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp

@@ -2446,17 +2446,11 @@
 #   error "__cpp_lib_associative_heterogeneous_erasure should not be defined before c++2b"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_assume_aligned
-#     error "__cpp_lib_assume_aligned should be defined in c++20"
-#   endif
-#   if __cpp_lib_assume_aligned != 201811L
-#     error "__cpp_lib_assume_aligned should have the value 201811L in c++20"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_assume_aligned
-#     error "__cpp_lib_assume_aligned should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_assume_aligned
+#   error "__cpp_lib_assume_aligned should be defined in c++20"
+# endif
+# if __cpp_lib_assume_aligned != 201811L
+#   error "__cpp_lib_assume_aligned should have the value 201811L in c++20"
 # endif
 
 # ifndef __cpp_lib_atomic_flag_test
@@ -3669,17 +3663,11 @@
 #   endif
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_assume_aligned
-#     error "__cpp_lib_assume_aligned should be defined in c++2b"
-#   endif
-#   if __cpp_lib_assume_aligned != 201811L
-#     error "__cpp_lib_assume_aligned should have the value 201811L in c++2b"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_assume_aligned
-#     error "__cpp_lib_assume_aligned should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_assume_aligned
+#   error "__cpp_lib_assume_aligned should be defined in c++2b"
+# endif
+# if __cpp_lib_assume_aligned != 201811L
+#   error "__cpp_lib_assume_aligned should have the value 201811L in c++2b"
 # endif
 
 # ifndef __cpp_lib_atomic_flag_test

libcxx/test/std/utilities/memory/ptr.align/assume_aligned.nodiscard.verify.cpp

@@ -0,0 +1,22 @@
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
+
+// #include <memory>
+
+// template<size_t N, class T>
+// [[nodiscard]] constexpr T* assume_aligned(T* ptr);
+
+#include <memory>
+
+void f() {
+  int *p = nullptr;
+  std::assume_aligned<4>(p); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+}

libcxx/test/std/utilities/memory/ptr.align/assume_aligned.pass.cpp

@@ -0,0 +1,88 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// #include <memory>
+
+// template<size_t N, class T>
+// [[nodiscard]] constexpr T* assume_aligned(T* ptr);
+
+#include <memory>
+#include <cassert>
+#include <cstddef>
+
+#include "test_macros.h"
+
+template <typename T>
+constexpr void check(T* p) {
+  ASSERT_SAME_TYPE(T*, decltype(std::assume_aligned<1>(p)));
+  constexpr std::size_t alignment = alignof(T);
+
+  if constexpr (alignment >= 1)
+    assert(p == std::assume_aligned<1>(p));
+  if constexpr (alignment >= 2)
+    assert(p == std::assume_aligned<2>(p));
+  if constexpr (alignment >= 4)
+    assert(p == std::assume_aligned<4>(p));
+  if constexpr (alignment >= 8)
+    assert(p == std::assume_aligned<8>(p));
+  if constexpr (alignment >= 16)
+    assert(p == std::assume_aligned<16>(p));
+  if constexpr (alignment >= 32)
+    assert(p == std::assume_aligned<32>(p));
+  if constexpr (alignment >= 64)
+    assert(p == std::assume_aligned<64>(p));
+  if constexpr (alignment >= 128)
+    assert(p == std::assume_aligned<128>(p));
+}
+
+struct              S    { };
+struct alignas(  4) S4   { };
+struct alignas(  8) S8   { };
+struct alignas( 16) S16  { };
+struct alignas( 32) S32  { };
+struct alignas( 64) S64  { };
+struct alignas(128) S128 { };
+
+constexpr bool tests() {
+  char        c;
+  int         i;
+  long        l;
+  double      d;
+  long double ld;
+  check( &c);
+  check( &i);
+  check( &l);
+  check( &d);
+  check(&ld);
+
+  S    s;
+  S4   s4;
+  S8   s8;
+  S16  s16;
+  S32  s32;
+  S64  s64;
+  S128 s128;
+  check(&s);
+  check(&s4);
+  check(&s8);
+  check(&s16);
+  check(&s32);
+  check(&s64);
+  check(&s128);
+
+  return true;
+}
+
+int main(int, char**) {
+  tests();
+  static_assert(tests());
+
+  return 0;
+}

libcxx/utils/generate_feature_test_macro_components.py

@@ -103,7 +103,6 @@ feature_test_macros = [ add_version_header(x) for x in [
     "name": "__cpp_lib_assume_aligned",
     "values": { "c++20": 201811 },
     "headers": ["memory"],
-    "unimplemented": True,
   }, {
     "name": "__cpp_lib_atomic_flag_test",
     "values": { "c++20": 201907 },