diff --git a/lib/libcxx/libc/include/llvm-libc-macros/stdfix-macros.h b/lib/libcxx/libc/include/llvm-libc-macros/stdfix-macros.h index 554ebe544a..04097e14e9 100644 --- a/lib/libcxx/libc/include/llvm-libc-macros/stdfix-macros.h +++ b/lib/libcxx/libc/include/llvm-libc-macros/stdfix-macros.h @@ -323,6 +323,45 @@ #define ULACCUM_EPSILON 0x1.0p-32ULK #endif // ULACCUM_EPSILON +#define absfx(x) \ + _Generic((x), \ + fract: absr, \ + short fract: abshr, \ + long fract: abslr, \ + accum: absk, \ + short accum: abshk, \ + long accum: abslk)(x) + +#define countlsfx(x) \ + _Generic((x), \ + fract: countlsr, \ + short fract: countlshr, \ + long fract: countlslr, \ + accum: countlsk, \ + short accum: countlshk, \ + long accum: countlslk, \ + unsigned fract: countlsur, \ + unsigned short fract: countlsuhr, \ + unsigned long fract: countlsulr, \ + unsigned accum: countlsuk, \ + unsigned short accum: countlsuhk, \ + unsigned long accum: countlsulk)(x) + +#define roundfx(x, y) \ + _Generic((x), \ + fract: roundr, \ + short fract: roundhr, \ + long fract: roundlr, \ + accum: roundk, \ + short accum: roundhk, \ + long accum: roundlk, \ + unsigned fract: roundur, \ + unsigned short fract: rounduhr, \ + unsigned long fract: roundulr, \ + unsigned accum: rounduk, \ + unsigned short accum: rounduhk, \ + unsigned long accum: roundulk)(x, y) + #endif // LIBC_COMPILER_HAS_FIXED_POINT #endif // LLVM_LIBC_MACROS_STDFIX_MACROS_H diff --git a/lib/libcxx/libc/shared/fp_bits.h b/lib/libcxx/libc/shared/fp_bits.h index 2898c508b7..e6bb1e17b8 100644 --- a/lib/libcxx/libc/shared/fp_bits.h +++ b/lib/libcxx/libc/shared/fp_bits.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIBC_SHARED_FP_BITS_H #define LLVM_LIBC_SHARED_FP_BITS_H +#include "libc_common.h" #include "src/__support/FPUtil/FPBits.h" namespace LIBC_NAMESPACE_DECL { diff --git a/lib/libcxx/libc/shared/libc_common.h b/lib/libcxx/libc/shared/libc_common.h new file mode 100644 index 0000000000..c4560bbb02 --- /dev/null +++ b/lib/libcxx/libc/shared/libc_common.h @@ -0,0 +1,26 @@ +//===-- Common defines for sharing LLVM libc with LLVM projects -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SHARED_LIBC_COMMON_H +#define LLVM_LIBC_SHARED_LIBC_COMMON_H + +// Use system errno. +#ifdef LIBC_ERRNO_MODE +#if LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_SYSTEM_INLINE +#error \ + "LIBC_ERRNO_MODE was set to something different from LIBC_ERRNO_MODE_SYSTEM_INLINE." +#endif // LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_SYSTEM_INLINE +#else +#define LIBC_ERRNO_MODE LIBC_ERRNO_MODE_SYSTEM_INLINE +#endif // LIBC_ERRNO_MODE + +#ifndef LIBC_NAMESPACE +#define LIBC_NAMESPACE __llvm_libc +#endif // LIBC_NAMESPACE + +#endif // LLVM_LIBC_SHARED_LIBC_COMMON_H diff --git a/lib/libcxx/libc/shared/str_to_float.h b/lib/libcxx/libc/shared/str_to_float.h index b133a28e26..dcc6027d6c 100644 --- a/lib/libcxx/libc/shared/str_to_float.h +++ b/lib/libcxx/libc/shared/str_to_float.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIBC_SHARED_STR_TO_FLOAT_H #define LLVM_LIBC_SHARED_STR_TO_FLOAT_H +#include "libc_common.h" #include "src/__support/str_to_float.h" namespace LIBC_NAMESPACE_DECL { diff --git a/lib/libcxx/libc/shared/str_to_integer.h b/lib/libcxx/libc/shared/str_to_integer.h index 15bee698d5..6ed38c9326 100644 --- a/lib/libcxx/libc/shared/str_to_integer.h +++ b/lib/libcxx/libc/shared/str_to_integer.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIBC_SHARED_STR_TO_INTEGER_H #define LLVM_LIBC_SHARED_STR_TO_INTEGER_H +#include "libc_common.h" #include "src/__support/str_to_integer.h" namespace LIBC_NAMESPACE_DECL { diff --git a/lib/libcxx/libc/src/__support/CPP/bit.h b/lib/libcxx/libc/src/__support/CPP/bit.h index adcd047274..e491f3e032 100644 --- a/lib/libcxx/libc/src/__support/CPP/bit.h +++ b/lib/libcxx/libc/src/__support/CPP/bit.h @@ -101,7 +101,7 @@ countr_zero(T value) { shift >>= 1; mask >>= shift; } - return zero_bits; + return static_cast(zero_bits); } #if __has_builtin(__builtin_ctzs) ADD_SPECIALIZATION(countr_zero, unsigned short, __builtin_ctzs) @@ -140,7 +140,7 @@ countl_zero(T value) { else zero_bits |= shift; } - return zero_bits; + return static_cast(zero_bits); } #if __has_builtin(__builtin_clzs) ADD_SPECIALIZATION(countl_zero, unsigned short, __builtin_clzs) @@ -162,7 +162,7 @@ ADD_SPECIALIZATION(countl_zero, unsigned long long, __builtin_clzll) template [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> countl_one(T value) { - return cpp::countl_zero(~value); + return cpp::countl_zero(static_cast(~value)); } /// Count the number of ones from the least significant bit to the first @@ -175,7 +175,7 @@ countl_one(T value) { template [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> countr_one(T value) { - return cpp::countr_zero(~value); + return cpp::countr_zero(static_cast(~value)); } /// Returns the number of bits needed to represent value if value is nonzero. @@ -226,25 +226,25 @@ rotr(T value, int rotate); template [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> rotl(T value, int rotate) { - constexpr unsigned N = cpp::numeric_limits::digits; + constexpr int N = cpp::numeric_limits::digits; rotate = rotate % N; if (!rotate) return value; if (rotate < 0) return cpp::rotr(value, -rotate); - return (value << rotate) | (value >> (N - rotate)); + return static_cast((value << rotate) | (value >> (N - rotate))); } template [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> rotr(T value, int rotate) { - constexpr unsigned N = cpp::numeric_limits::digits; + constexpr int N = cpp::numeric_limits::digits; rotate = rotate % N; if (!rotate) return value; if (rotate < 0) return cpp::rotl(value, -rotate); - return (value >> rotate) | (value << (N - rotate)); + return static_cast((value >> rotate) | (value << (N - rotate))); } // TODO: Do we need this function at all? How is it different from diff --git a/lib/libcxx/libc/src/__support/CPP/string_view.h b/lib/libcxx/libc/src/__support/CPP/string_view.h index 745c62c35f..aa15814b2e 100644 --- a/lib/libcxx/libc/src/__support/CPP/string_view.h +++ b/lib/libcxx/libc/src/__support/CPP/string_view.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIBC_SRC___SUPPORT_CPP_STRING_VIEW_H #define LLVM_LIBC_SRC___SUPPORT_CPP_STRING_VIEW_H +#include "limits.h" #include "src/__support/common.h" #include "src/__support/macros/config.h" @@ -40,7 +41,7 @@ private: LIBC_INLINE static constexpr size_t length(const char *Str) { for (const char *End = Str;; ++End) if (*End == '\0') - return End - Str; + return static_cast(End - Str); } LIBC_INLINE bool equals(string_view Other) const { @@ -61,7 +62,8 @@ public: // special value equal to the maximum value representable by the type // size_type. - LIBC_INLINE_VAR static constexpr size_t npos = -1; + LIBC_INLINE_VAR static constexpr size_t npos = + cpp::numeric_limits::max(); LIBC_INLINE constexpr string_view() : Data(nullptr), Len(0) {} diff --git a/lib/libcxx/libc/src/__support/CPP/type_traits/is_floating_point.h b/lib/libcxx/libc/src/__support/CPP/type_traits/is_floating_point.h index 11ffbfabe6..9dc77ad7ee 100644 --- a/lib/libcxx/libc/src/__support/CPP/type_traits/is_floating_point.h +++ b/lib/libcxx/libc/src/__support/CPP/type_traits/is_floating_point.h @@ -36,7 +36,8 @@ public: , float128 #endif - >(); + , + bfloat16>(); }; template LIBC_INLINE_VAR constexpr bool is_floating_point_v = diff --git a/lib/libcxx/libc/src/__support/CPP/type_traits/is_signed.h b/lib/libcxx/libc/src/__support/CPP/type_traits/is_signed.h index 3f56fb38aa..2ddb43ac4e 100644 --- a/lib/libcxx/libc/src/__support/CPP/type_traits/is_signed.h +++ b/lib/libcxx/libc/src/__support/CPP/type_traits/is_signed.h @@ -8,20 +8,43 @@ #ifndef LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_SIGNED_H #define LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_SIGNED_H +#include "include/llvm-libc-macros/stdfix-macros.h" #include "src/__support/CPP/type_traits/bool_constant.h" #include "src/__support/CPP/type_traits/is_arithmetic.h" +#include "src/__support/CPP/type_traits/is_same.h" +#include "src/__support/CPP/type_traits/remove_cv.h" #include "src/__support/macros/attributes.h" #include "src/__support/macros/config.h" namespace LIBC_NAMESPACE_DECL { namespace cpp { -// is_signed +#ifndef LIBC_COMPILER_HAS_FIXED_POINT template struct is_signed : bool_constant<(is_arithmetic_v && (T(-1) < T(0)))> { LIBC_INLINE constexpr operator bool() const { return is_signed::value; } LIBC_INLINE constexpr bool operator()() const { return is_signed::value; } }; +#else +template struct is_signed { +private: + template + LIBC_INLINE static constexpr bool __is_unqualified_any_of() { + return (... || is_same_v, Args>); + } + +public: + LIBC_INLINE_VAR static constexpr bool value = + (is_arithmetic_v && (T(-1) < T(0))) || + __is_unqualified_any_of(); + LIBC_INLINE constexpr operator bool() const { return is_signed::value; } + LIBC_INLINE constexpr bool operator()() const { return is_signed::value; } +}; +#endif // LIBC_COMPILER_HAS_FIXED_POINT + template LIBC_INLINE_VAR constexpr bool is_signed_v = is_signed::value; diff --git a/lib/libcxx/libc/src/__support/CPP/type_traits/is_unsigned.h b/lib/libcxx/libc/src/__support/CPP/type_traits/is_unsigned.h index eed519b1c0..3ae6337ceb 100644 --- a/lib/libcxx/libc/src/__support/CPP/type_traits/is_unsigned.h +++ b/lib/libcxx/libc/src/__support/CPP/type_traits/is_unsigned.h @@ -8,20 +8,45 @@ #ifndef LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_UNSIGNED_H #define LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_UNSIGNED_H +#include "include/llvm-libc-macros/stdfix-macros.h" #include "src/__support/CPP/type_traits/bool_constant.h" #include "src/__support/CPP/type_traits/is_arithmetic.h" +#include "src/__support/CPP/type_traits/is_same.h" +#include "src/__support/CPP/type_traits/remove_cv.h" #include "src/__support/macros/attributes.h" #include "src/__support/macros/config.h" namespace LIBC_NAMESPACE_DECL { namespace cpp { -// is_unsigned +#ifndef LIBC_COMPILER_HAS_FIXED_POINT template struct is_unsigned : bool_constant<(is_arithmetic_v && (T(-1) > T(0)))> { LIBC_INLINE constexpr operator bool() const { return is_unsigned::value; } LIBC_INLINE constexpr bool operator()() const { return is_unsigned::value; } }; +#else +template struct is_unsigned { +private: + template + LIBC_INLINE static constexpr bool __is_unqualified_any_of() { + return (... || is_same_v, Args>); + } + +public: + LIBC_INLINE_VAR static constexpr bool value = + (is_arithmetic_v && (T(-1) > T(0))) || + __is_unqualified_any_of(); + LIBC_INLINE constexpr operator bool() const { return is_unsigned::value; } + LIBC_INLINE constexpr bool operator()() const { return is_unsigned::value; } +}; +#endif // LIBC_COMPILER_HAS_FIXED_POINT + template LIBC_INLINE_VAR constexpr bool is_unsigned_v = is_unsigned::value; diff --git a/lib/libcxx/libc/src/__support/FPUtil/FPBits.h b/lib/libcxx/libc/src/__support/FPUtil/FPBits.h index 90b6e406e0..9e21136558 100644 --- a/lib/libcxx/libc/src/__support/FPUtil/FPBits.h +++ b/lib/libcxx/libc/src/__support/FPUtil/FPBits.h @@ -38,6 +38,7 @@ enum class FPType { IEEE754_Binary64, IEEE754_Binary128, X86_Binary80, + BFloat16 }; // The classes hierarchy is as follows: @@ -138,6 +139,14 @@ template <> struct FPLayout { LIBC_INLINE_VAR static constexpr int FRACTION_LEN = SIG_LEN - 1; }; +template <> struct FPLayout { + using StorageType = uint16_t; + LIBC_INLINE_VAR static constexpr int SIGN_LEN = 1; + LIBC_INLINE_VAR static constexpr int EXP_LEN = 8; + LIBC_INLINE_VAR static constexpr int SIG_LEN = 7; + LIBC_INLINE_VAR static constexpr int FRACTION_LEN = SIG_LEN; +}; + // FPStorage derives useful constants from the FPLayout above. template struct FPStorage : public FPLayout { using UP = FPLayout; @@ -247,11 +256,11 @@ protected: using UP::UP; LIBC_INLINE constexpr BiasedExponent(Exponent exp) - : UP(static_cast(exp) + EXP_BIAS) {} + : UP(static_cast(static_cast(exp) + EXP_BIAS)) {} // Cast operator to get convert from BiasedExponent to Exponent. LIBC_INLINE constexpr operator Exponent() const { - return Exponent(UP::value - EXP_BIAS); + return Exponent(static_cast(UP::value - EXP_BIAS)); } LIBC_INLINE constexpr BiasedExponent &operator++() { @@ -686,7 +695,7 @@ public: } LIBC_INLINE constexpr void set_biased_exponent(StorageType biased) { - UP::set_biased_exponent(BiasedExponent((int32_t)biased)); + UP::set_biased_exponent(BiasedExponent(static_cast(biased))); } LIBC_INLINE constexpr int get_exponent() const { @@ -757,7 +766,7 @@ public: result.set_significand(number); result.set_biased_exponent(static_cast(ep + 1)); } else { - result.set_significand(number >> -ep); + result.set_significand(number >> static_cast(-ep)); } return RetT(result.uintval()); } @@ -801,6 +810,8 @@ template LIBC_INLINE static constexpr FPType get_fp_type() { else if constexpr (cpp::is_same_v) return FPType::IEEE754_Binary128; #endif + else if constexpr (cpp::is_same_v) + return FPType::BFloat16; else static_assert(cpp::always_false, "Unsupported type"); } diff --git a/lib/libcxx/libc/src/__support/big_int.h b/lib/libcxx/libc/src/__support/big_int.h index a95ab4ff8e..85db31d013 100644 --- a/lib/libcxx/libc/src/__support/big_int.h +++ b/lib/libcxx/libc/src/__support/big_int.h @@ -241,7 +241,7 @@ LIBC_INLINE constexpr void quick_mul_hi(cpp::array &dst, } template -LIBC_INLINE constexpr bool is_negative(cpp::array &array) { +LIBC_INLINE constexpr bool is_negative(const cpp::array &array) { using signed_word = cpp::make_signed_t; return cpp::bit_cast(array.back()) < 0; } @@ -284,8 +284,8 @@ LIBC_INLINE constexpr cpp::array shift(cpp::array array, if (i < 0) return 0; if (i >= int(N)) - return is_neg ? -1 : 0; - return array[i]; + return is_neg ? cpp::numeric_limits::max() : 0; + return array[static_cast(i)]; }; const size_t index_offset = offset / WORD_BITS; const size_t bit_offset = offset % WORD_BITS; @@ -296,7 +296,7 @@ LIBC_INLINE constexpr cpp::array shift(cpp::array array, for (size_t index = 0; index < N; ++index) { const word part1 = safe_get_at(index + index_offset); const word part2 = safe_get_at(index + index_offset + 1); - word &dst = out[at(index)]; + word &dst = out[static_cast(at(index))]; if (bit_offset == 0) dst = part1; // no crosstalk between parts. else if constexpr (direction == LEFT) @@ -465,8 +465,7 @@ public: } // Initialize the first word to |v| and the rest to 0. - template && - !cpp::is_same_v>> + template >> LIBC_INLINE constexpr BigInt(T v) { constexpr size_t T_SIZE = sizeof(T) * CHAR_BIT; const bool is_neg = v < 0; @@ -697,7 +696,8 @@ public: } BigInt quotient; WordType x_word = static_cast(x); - constexpr size_t LOG2_WORD_SIZE = cpp::bit_width(WORD_SIZE) - 1; + constexpr size_t LOG2_WORD_SIZE = + static_cast(cpp::bit_width(WORD_SIZE) - 1); constexpr size_t HALF_WORD_SIZE = WORD_SIZE >> 1; constexpr WordType HALF_MASK = ((WordType(1) << HALF_WORD_SIZE) - 1); // lower = smallest multiple of WORD_SIZE that is >= e. @@ -866,7 +866,7 @@ public: LIBC_INLINE constexpr BigInt operator~() const { BigInt result; for (size_t i = 0; i < WORD_COUNT; ++i) - result[i] = ~val[i]; + result[i] = static_cast(~val[i]); return result; } @@ -936,6 +936,18 @@ public: // Return the i-th word of the number. LIBC_INLINE constexpr WordType &operator[](size_t i) { return val[i]; } + // Return the i-th bit of the number. + LIBC_INLINE constexpr bool get_bit(size_t i) const { + const size_t word_index = i / WORD_SIZE; + return 1 & (val[word_index] >> (i % WORD_SIZE)); + } + + // Set the i-th bit of the number. + LIBC_INLINE constexpr void set_bit(size_t i) { + const size_t word_index = i / WORD_SIZE; + val[word_index] |= WordType(1) << (i % WORD_SIZE); + } + private: LIBC_INLINE friend constexpr int cmp(const BigInt &lhs, const BigInt &rhs) { constexpr auto compare = [](WordType a, WordType b) { @@ -955,7 +967,7 @@ private: LIBC_INLINE constexpr void bitwise_not() { for (auto &part : val) - part = ~part; + part = static_cast(~part); } LIBC_INLINE constexpr void negate() { @@ -968,7 +980,7 @@ private: } LIBC_INLINE constexpr void decrement() { - multiword::add_with_carry(val, cpp::array{1}); + multiword::sub_with_borrow(val, cpp::array{1}); } LIBC_INLINE constexpr void extend(size_t index, bool is_neg) { @@ -989,12 +1001,6 @@ private: LIBC_INLINE constexpr void clear_msb() { val.back() &= mask_trailing_ones(); } - - LIBC_INLINE constexpr void set_bit(size_t i) { - const size_t word_index = i / WORD_SIZE; - val[word_index] |= WordType(1) << (i % WORD_SIZE); - } - LIBC_INLINE constexpr static Division divide_unsigned(const BigInt ÷nd, const BigInt ÷r) { BigInt remainder = dividend; @@ -1003,12 +1009,12 @@ private: BigInt subtractor = divider; int cur_bit = multiword::countl_zero(subtractor.val) - multiword::countl_zero(remainder.val); - subtractor <<= cur_bit; + subtractor <<= static_cast(cur_bit); for (; cur_bit >= 0 && remainder > 0; --cur_bit, subtractor >>= 1) { if (remainder < subtractor) continue; remainder -= subtractor; - quotient.set_bit(cur_bit); + quotient.set_bit(static_cast(cur_bit)); } } return Division{quotient, remainder}; @@ -1270,26 +1276,28 @@ rotr(T value, int rotate); template [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> rotl(T value, int rotate) { - constexpr unsigned N = cpp::numeric_limits::digits; + constexpr int N = cpp::numeric_limits::digits; rotate = rotate % N; if (!rotate) return value; if (rotate < 0) return cpp::rotr(value, -rotate); - return (value << rotate) | (value >> (N - rotate)); + return (value << static_cast(rotate)) | + (value >> (N - static_cast(rotate))); } // Specialization of cpp::rotr ('bit.h') for BigInt. template [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> rotr(T value, int rotate) { - constexpr unsigned N = cpp::numeric_limits::digits; + constexpr int N = cpp::numeric_limits::digits; rotate = rotate % N; if (!rotate) return value; if (rotate < 0) return cpp::rotl(value, -rotate); - return (value >> rotate) | (value << (N - rotate)); + return (value >> static_cast(rotate)) | + (value << (N - static_cast(rotate))); } } // namespace cpp @@ -1306,7 +1314,7 @@ mask_trailing_ones() { T out; // zero initialized for (size_t i = 0; i <= QUOTIENT; ++i) out[i] = i < QUOTIENT - ? -1 + ? cpp::numeric_limits::max() : mask_trailing_ones(); return out; } @@ -1322,7 +1330,7 @@ LIBC_INLINE constexpr cpp::enable_if_t, T> mask_leading_ones() { T out; // zero initialized for (size_t i = QUOTIENT; i < T::WORD_COUNT; ++i) out[i] = i > QUOTIENT - ? -1 + ? cpp::numeric_limits::max() : mask_leading_ones(); return out; } @@ -1375,8 +1383,7 @@ first_trailing_zero(T value) { template [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> first_trailing_one(T value) { - return value == cpp::numeric_limits::max() ? 0 - : cpp::countr_zero(value) + 1; + return value == 0 ? 0 : cpp::countr_zero(value) + 1; } } // namespace LIBC_NAMESPACE_DECL diff --git a/lib/libcxx/libc/src/__support/common.h b/lib/libcxx/libc/src/__support/common.h index 42e8a79187..15209b7697 100644 --- a/lib/libcxx/libc/src/__support/common.h +++ b/lib/libcxx/libc/src/__support/common.h @@ -37,17 +37,27 @@ #define LLVM_LIBC_ATTR(name) EXPAND_THEN_SECOND(LLVM_LIBC_FUNCTION_ATTR_##name) -// MacOS needs to be excluded because it does not support aliasing. -#if defined(LIBC_COPT_PUBLIC_PACKAGING) && (!defined(__APPLE__)) +// At the moment, [[gnu::alias()]] is not supported on MacOS, and it is needed +// to cleanly export and alias the C++ symbol `LIBC_NAMESPACE::func` with the C +// symbol `func`. So for public packaging on MacOS, we will only export the C +// symbol. Moreover, a C symbol `func` in macOS is mangled as `_func`. +#if defined(LIBC_COPT_PUBLIC_PACKAGING) +#ifndef __APPLE__ #define LLVM_LIBC_FUNCTION_IMPL(type, name, arglist) \ LLVM_LIBC_ATTR(name) \ LLVM_LIBC_FUNCTION_ATTR decltype(LIBC_NAMESPACE::name) \ __##name##_impl__ __asm__(#name); \ decltype(LIBC_NAMESPACE::name) name [[gnu::alias(#name)]]; \ type __##name##_impl__ arglist -#else +#else // __APPLE__ +#define LLVM_LIBC_FUNCTION_IMPL(type, name, arglist) \ + LLVM_LIBC_ATTR(name) \ + LLVM_LIBC_FUNCTION_ATTR decltype(LIBC_NAMESPACE::name) name asm("_" #name); \ + type name arglist +#endif // __APPLE__ +#else // LIBC_COPT_PUBLIC_PACKAGING #define LLVM_LIBC_FUNCTION_IMPL(type, name, arglist) type name arglist -#endif +#endif // LIBC_COPT_PUBLIC_PACKAGING // This extra layer of macro allows `name` to be a macro to rename a function. #define LLVM_LIBC_FUNCTION(type, name, arglist) \ diff --git a/lib/libcxx/libc/src/__support/high_precision_decimal.h b/lib/libcxx/libc/src/__support/high_precision_decimal.h index 922dce484a..cb4b50c315 100644 --- a/lib/libcxx/libc/src/__support/high_precision_decimal.h +++ b/lib/libcxx/libc/src/__support/high_precision_decimal.h @@ -264,7 +264,7 @@ private: LIBC_INLINE void left_shift(uint32_t shift_amount) { uint32_t new_digits = this->get_num_new_digits(shift_amount); - int32_t read_index = this->num_digits - 1; + int32_t read_index = static_cast(this->num_digits - 1); uint32_t write_index = this->num_digits + new_digits; uint64_t accumulator = 0; @@ -329,7 +329,7 @@ public: if (saw_dot) { break; } - this->decimal_point = total_digits; + this->decimal_point = static_cast(total_digits); saw_dot = true; } else { if (num_string[num_cur] == '0' && this->num_digits == 0) { @@ -350,7 +350,7 @@ public: } if (!saw_dot) - this->decimal_point = total_digits; + this->decimal_point = static_cast(total_digits); if (num_cur < num_len && (num_string[num_cur] == 'e' || num_string[num_cur] == 'E')) { @@ -393,7 +393,7 @@ public: this->left_shift(MAX_SHIFT_AMOUNT); shift_amount -= MAX_SHIFT_AMOUNT; } - this->left_shift(shift_amount); + this->left_shift(static_cast(shift_amount)); } // Right else { @@ -401,7 +401,7 @@ public: this->right_shift(MAX_SHIFT_AMOUNT); shift_amount += MAX_SHIFT_AMOUNT; } - this->right_shift(-shift_amount); + this->right_shift(static_cast(-shift_amount)); } } @@ -424,8 +424,8 @@ public: result *= 10; ++cur_digit; } - return result + static_cast( - this->should_round_up(this->decimal_point, round)); + return result + + static_cast(this->should_round_up(this->decimal_point, round)); } // Extra functions for testing. diff --git a/lib/libcxx/libc/src/__support/libc_assert.h b/lib/libcxx/libc/src/__support/libc_assert.h index 3db179ff67..ada1795ccb 100644 --- a/lib/libcxx/libc/src/__support/libc_assert.h +++ b/lib/libcxx/libc/src/__support/libc_assert.h @@ -9,7 +9,6 @@ #ifndef LLVM_LIBC_SRC___SUPPORT_LIBC_ASSERT_H #define LLVM_LIBC_SRC___SUPPORT_LIBC_ASSERT_H -#include "src/__support/macros/config.h" #if defined(LIBC_COPT_USE_C_ASSERT) || !defined(LIBC_FULL_BUILD) // The build is configured to just use the public API @@ -25,6 +24,7 @@ #include "src/__support/OSUtil/io.h" #include "src/__support/integer_to_string.h" #include "src/__support/macros/attributes.h" // For LIBC_INLINE +#include "src/__support/macros/config.h" #include "src/__support/macros/optimization.h" // For LIBC_UNLIKELY namespace LIBC_NAMESPACE_DECL { diff --git a/lib/libcxx/libc/src/__support/macros/null_check.h b/lib/libcxx/libc/src/__support/macros/null_check.h index eda19f8892..abf65c56c4 100644 --- a/lib/libcxx/libc/src/__support/macros/null_check.h +++ b/lib/libcxx/libc/src/__support/macros/null_check.h @@ -19,10 +19,19 @@ if (LIBC_UNLIKELY((ptr) == nullptr)) \ __builtin_trap(); \ } while (0) +#define LIBC_CRASH_ON_VALUE(var, value) \ + do { \ + if (LIBC_UNLIKELY((var) == (value))) \ + __builtin_trap(); \ + } while (0) + #else #define LIBC_CRASH_ON_NULLPTR(ptr) \ do { \ } while (0) +#define LIBC_CRASH_ON_VALUE(var, value) \ + do { \ + } while (0) #endif #endif // LLVM_LIBC_SRC___SUPPORT_MACROS_NULL_CHECK_H diff --git a/lib/libcxx/libc/src/__support/macros/optimization.h b/lib/libcxx/libc/src/__support/macros/optimization.h index a2634950d4..250a9e0607 100644 --- a/lib/libcxx/libc/src/__support/macros/optimization.h +++ b/lib/libcxx/libc/src/__support/macros/optimization.h @@ -10,7 +10,7 @@ #ifndef LLVM_LIBC_SRC___SUPPORT_MACROS_OPTIMIZATION_H #define LLVM_LIBC_SRC___SUPPORT_MACROS_OPTIMIZATION_H -#include "src/__support/macros/attributes.h" // LIBC_INLINE +#include "src/__support/macros/attributes.h" // LIBC_INLINE #include "src/__support/macros/config.h" #include "src/__support/macros/properties/compiler.h" // LIBC_COMPILER_IS_CLANG @@ -30,8 +30,10 @@ LIBC_INLINE constexpr bool expects_bool_condition(T value, T expected) { #if defined(LIBC_COMPILER_IS_CLANG) #define LIBC_LOOP_NOUNROLL _Pragma("nounroll") +#define LIBC_LOOP_UNROLL _Pragma("unroll") #elif defined(LIBC_COMPILER_IS_GCC) #define LIBC_LOOP_NOUNROLL _Pragma("GCC unroll 0") +#define LIBC_LOOP_UNROLL _Pragma("GCC unroll 2048") #else #error "Unhandled compiler" #endif @@ -45,6 +47,7 @@ LIBC_INLINE constexpr bool expects_bool_condition(T value, T expected) { #define LIBC_MATH_FAST \ (LIBC_MATH_SKIP_ACCURATE_PASS | LIBC_MATH_SMALL_TABLES | \ LIBC_MATH_NO_ERRNO | LIBC_MATH_NO_EXCEPT) +#define LIBC_MATH_INTERMEDIATE_COMP_IN_FLOAT 0x10 #ifndef LIBC_MATH #define LIBC_MATH 0 @@ -58,4 +61,16 @@ LIBC_INLINE constexpr bool expects_bool_condition(T value, T expected) { #define LIBC_MATH_HAS_SMALL_TABLES #endif +#if (LIBC_MATH & LIBC_MATH_INTERMEDIATE_COMP_IN_FLOAT) +#define LIBC_MATH_HAS_INTERMEDIATE_COMP_IN_FLOAT +#endif + +#if (LIBC_MATH & LIBC_MATH_NO_ERRNO) +#define LIBC_MATH_HAS_NO_ERRNO +#endif + +#if (LIBC_MATH & LIBC_MATH_NO_EXCEPT) +#define LIBC_MATH_HAS_NO_EXCEPT +#endif + #endif // LLVM_LIBC_SRC___SUPPORT_MACROS_OPTIMIZATION_H diff --git a/lib/libcxx/libc/src/__support/macros/properties/cpu_features.h b/lib/libcxx/libc/src/__support/macros/properties/cpu_features.h index d2cea36751..cdb2df97b2 100644 --- a/lib/libcxx/libc/src/__support/macros/properties/cpu_features.h +++ b/lib/libcxx/libc/src/__support/macros/properties/cpu_features.h @@ -20,6 +20,8 @@ #if defined(__SSE2__) #define LIBC_TARGET_CPU_HAS_SSE2 +#define LIBC_TARGET_CPU_HAS_FPU_FLOAT +#define LIBC_TARGET_CPU_HAS_FPU_DOUBLE #endif #if defined(__SSE4_2__) @@ -42,9 +44,55 @@ #define LIBC_TARGET_CPU_HAS_AVX512BW #endif +#if defined(__ARM_FP) +#if (__ARM_FP & 0x2) +#define LIBC_TARGET_CPU_HAS_ARM_FPU_HALF +#define LIBC_TARGET_CPU_HAS_FPU_HALF +#endif // LIBC_TARGET_CPU_HAS_ARM_FPU_HALF +#if (__ARM_FP & 0x4) +#define LIBC_TARGET_CPU_HAS_ARM_FPU_FLOAT +#define LIBC_TARGET_CPU_HAS_FPU_FLOAT +#endif // LIBC_TARGET_CPU_HAS_ARM_FPU_FLOAT +#if (__ARM_FP & 0x8) +#define LIBC_TARGET_CPU_HAS_ARM_FPU_DOUBLE +#define LIBC_TARGET_CPU_HAS_FPU_DOUBLE +#endif // LIBC_TARGET_CPU_HAS_ARM_FPU_DOUBLE +#endif // __ARM_FP + +#if defined(__riscv_flen) +// https://github.com/riscv-non-isa/riscv-c-api-doc/blob/main/src/c-api.adoc +#if defined(__riscv_zfhmin) +#define LIBC_TARGET_CPU_HAS_RISCV_FPU_HALF +#define LIBC_TARGET_CPU_HAS_FPU_HALF +#endif // LIBC_TARGET_CPU_HAS_RISCV_FPU_HALF +#if (__riscv_flen >= 32) +#define LIBC_TARGET_CPU_HAS_RISCV_FPU_FLOAT +#define LIBC_TARGET_CPU_HAS_FPU_FLOAT +#endif // LIBC_TARGET_CPU_HAS_RISCV_FPU_FLOAT +#if (__riscv_flen >= 64) +#define LIBC_TARGET_CPU_HAS_RISCV_FPU_DOUBLE +#define LIBC_TARGET_CPU_HAS_FPU_DOUBLE +#endif // LIBC_TARGET_CPU_HAS_RISCV_FPU_DOUBLE +#endif // __riscv_flen + +#if defined(__NVPTX__) || defined(__AMDGPU__) +#define LIBC_TARGET_CPU_HAS_FPU_FLOAT +#define LIBC_TARGET_CPU_HAS_FPU_DOUBLE +#endif + #if defined(__ARM_FEATURE_FMA) || (defined(__AVX2__) && defined(__FMA__)) || \ defined(__NVPTX__) || defined(__AMDGPU__) || defined(__LIBC_RISCV_USE_FMA) #define LIBC_TARGET_CPU_HAS_FMA +// Provide a more fine-grained control of FMA instruction for ARM targets. +#if defined(LIBC_TARGET_CPU_HAS_FPU_HALF) +#define LIBC_TARGET_CPU_HAS_FMA_HALF +#endif // LIBC_TARGET_CPU_HAS_FMA_HALF +#if defined(LIBC_TARGET_CPU_HAS_FPU_FLOAT) +#define LIBC_TARGET_CPU_HAS_FMA_FLOAT +#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT +#if defined(LIBC_TARGET_CPU_HAS_FPU_DOUBLE) +#define LIBC_TARGET_CPU_HAS_FMA_DOUBLE +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE #endif #if defined(LIBC_TARGET_ARCH_IS_AARCH64) || \ diff --git a/lib/libcxx/libc/src/__support/macros/properties/types.h b/lib/libcxx/libc/src/__support/macros/properties/types.h index 6293b9d4d2..aec4a488c3 100644 --- a/lib/libcxx/libc/src/__support/macros/properties/types.h +++ b/lib/libcxx/libc/src/__support/macros/properties/types.h @@ -10,9 +10,10 @@ #ifndef LLVM_LIBC_SRC___SUPPORT_MACROS_PROPERTIES_TYPES_H #define LLVM_LIBC_SRC___SUPPORT_MACROS_PROPERTIES_TYPES_H -#include "hdr/float_macros.h" // LDBL_MANT_DIG +#include "hdr/float_macros.h" // LDBL_MANT_DIG #include "include/llvm-libc-macros/float16-macros.h" // LIBC_TYPES_HAS_FLOAT16 #include "include/llvm-libc-types/float128.h" // float128 +#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL #include "src/__support/macros/properties/architectures.h" #include "src/__support/macros/properties/compiler.h" #include "src/__support/macros/properties/cpu_features.h" @@ -58,4 +59,14 @@ using float16 = _Float16; // LIBC_TYPES_HAS_FLOAT128 and 'float128' type are provided by // "include/llvm-libc-types/float128.h" +// -- bfloat16 support --------------------------------------------------------- + +namespace LIBC_NAMESPACE_DECL { +namespace fputil { +struct BFloat16; +} +} // namespace LIBC_NAMESPACE_DECL + +using bfloat16 = LIBC_NAMESPACE::fputil::BFloat16; + #endif // LLVM_LIBC_SRC___SUPPORT_MACROS_PROPERTIES_TYPES_H diff --git a/lib/libcxx/libc/src/__support/math_extras.h b/lib/libcxx/libc/src/__support/math_extras.h index 6f4a006aad..47df2a4325 100644 --- a/lib/libcxx/libc/src/__support/math_extras.h +++ b/lib/libcxx/libc/src/__support/math_extras.h @@ -146,8 +146,7 @@ first_trailing_zero(T value) { template [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> first_trailing_one(T value) { - return value == cpp::numeric_limits::max() ? 0 - : cpp::countr_zero(value) + 1; + return value == 0 ? 0 : cpp::countr_zero(value) + 1; } template diff --git a/lib/libcxx/libc/src/__support/sign.h b/lib/libcxx/libc/src/__support/sign.h index 4a629e4488..e0de0e0798 100644 --- a/lib/libcxx/libc/src/__support/sign.h +++ b/lib/libcxx/libc/src/__support/sign.h @@ -29,6 +29,8 @@ struct Sign { static const Sign POS; static const Sign NEG; + LIBC_INLINE constexpr Sign negate() const { return Sign(!is_negative); } + private: LIBC_INLINE constexpr explicit Sign(bool is_negative) : is_negative(is_negative) {} diff --git a/lib/libcxx/libc/src/__support/str_to_float.h b/lib/libcxx/libc/src/__support/str_to_float.h index b4d5646822..a7dd7ce0ae 100644 --- a/lib/libcxx/libc/src/__support/str_to_float.h +++ b/lib/libcxx/libc/src/__support/str_to_float.h @@ -15,6 +15,7 @@ #ifndef LLVM_LIBC_SRC___SUPPORT_STR_TO_FLOAT_H #define LLVM_LIBC_SRC___SUPPORT_STR_TO_FLOAT_H +#include "hdr/errno_macros.h" // For ERANGE #include "src/__support/CPP/bit.h" #include "src/__support/CPP/limits.h" #include "src/__support/CPP/optional.h" @@ -31,7 +32,6 @@ #include "src/__support/str_to_integer.h" #include "src/__support/str_to_num_result.h" #include "src/__support/uint128.h" -#include "src/errno/libc_errno.h" // For ERANGE #include @@ -108,11 +108,11 @@ eisel_lemire(ExpandedFloat init_num, } // Normalization - uint32_t clz = cpp::countl_zero(mantissa); + uint32_t clz = static_cast(cpp::countl_zero(mantissa)); mantissa <<= clz; - int32_t exp2 = - exp10_to_exp2(exp10) + FPBits::STORAGE_LEN + FPBits::EXP_BIAS - clz; + int32_t exp2 = exp10_to_exp2(exp10) + FPBits::STORAGE_LEN + FPBits::EXP_BIAS - + static_cast(clz); // Multiplication const uint64_t *power_of_ten = @@ -225,8 +225,8 @@ eisel_lemire(ExpandedFloat init_num, } // Normalization - uint32_t clz = cpp::countl_zero(mantissa) - - ((sizeof(UInt128) - sizeof(StorageType)) * CHAR_BIT); + int32_t clz = static_cast(cpp::countl_zero(mantissa)) - + ((sizeof(UInt128) - sizeof(StorageType)) * CHAR_BIT); mantissa <<= clz; int32_t exp2 = @@ -802,7 +802,7 @@ LIBC_INLINE FloatConvertReturn binary_exp_to_float(ExpandedFloat init_num, // Handle subnormals. if (biased_exponent <= 0) { - amount_to_shift_right += 1 - biased_exponent; + amount_to_shift_right += static_cast(1 - biased_exponent); biased_exponent = 0; if (amount_to_shift_right > FPBits::STORAGE_LEN) { @@ -909,7 +909,7 @@ decimal_string_to_float(const char *__restrict src, const char DECIMAL_POINT, cpp::numeric_limits::max() / BASE; while (true) { if (isdigit(src[index])) { - uint32_t digit = b36_char_to_int(src[index]); + uint32_t digit = static_cast(b36_char_to_int(src[index])); seen_digit = true; if (mantissa < bitstype_max_div_by_base) { @@ -956,7 +956,7 @@ decimal_string_to_float(const char *__restrict src, const char DECIMAL_POINT, if (result.has_error()) output.error = result.error; int32_t add_to_exponent = result.value; - index += result.parsed_len; + index += static_cast(result.parsed_len); // Here we do this operation as int64 to avoid overflow. int64_t temp_exponent = static_cast(exponent) + @@ -1020,7 +1020,7 @@ hexadecimal_string_to_float(const char *__restrict src, cpp::numeric_limits::max() / BASE; while (true) { if (isalnum(src[index])) { - uint32_t digit = b36_char_to_int(src[index]); + uint32_t digit = static_cast(b36_char_to_int(src[index])); if (digit < BASE) seen_digit = true; else @@ -1070,7 +1070,7 @@ hexadecimal_string_to_float(const char *__restrict src, output.error = result.error; int32_t add_to_exponent = result.value; - index += result.parsed_len; + index += static_cast(result.parsed_len); // Here we do this operation as int64 to avoid overflow. int64_t temp_exponent = static_cast(exponent) + @@ -1135,7 +1135,7 @@ LIBC_INLINE StrToNumResult strtofloatingpoint(const char *__restrict src) { int error = 0; - ptrdiff_t index = first_non_whitespace(src) - src; + size_t index = first_non_whitespace(src); if (src[index] == '+' || src[index] == '-') { sign = src[index]; @@ -1245,7 +1245,7 @@ LIBC_INLINE StrToNumResult strtofloatingpoint(const char *__restrict src) { // special 80 bit long doubles. Otherwise it should be inlined out. set_implicit_bit(result); - return {result.get_val(), index, error}; + return {result.get_val(), static_cast(index), error}; } template LIBC_INLINE StrToNumResult strtonan(const char *arg) { diff --git a/lib/libcxx/libc/src/__support/str_to_integer.h b/lib/libcxx/libc/src/__support/str_to_integer.h index 8e569e8a7f..d332c929f2 100644 --- a/lib/libcxx/libc/src/__support/str_to_integer.h +++ b/lib/libcxx/libc/src/__support/str_to_integer.h @@ -15,6 +15,7 @@ #ifndef LLVM_LIBC_SRC___SUPPORT_STR_TO_INTEGER_H #define LLVM_LIBC_SRC___SUPPORT_STR_TO_INTEGER_H +#include "hdr/errno_macros.h" // For ERANGE #include "src/__support/CPP/limits.h" #include "src/__support/CPP/type_traits.h" #include "src/__support/CPP/type_traits/make_unsigned.h" @@ -24,22 +25,20 @@ #include "src/__support/macros/config.h" #include "src/__support/str_to_num_result.h" #include "src/__support/uint128.h" -#include "src/errno/libc_errno.h" // For ERANGE namespace LIBC_NAMESPACE_DECL { namespace internal { -// Returns a pointer to the first character in src that is not a whitespace +// Returns the idx to the first character in src that is not a whitespace // character (as determined by isspace()) -// TODO: Change from returning a pointer to returning a length. -LIBC_INLINE const char * +LIBC_INLINE size_t first_non_whitespace(const char *__restrict src, size_t src_len = cpp::numeric_limits::max()) { size_t src_cur = 0; while (src_cur < src_len && internal::isspace(src[src_cur])) { ++src_cur; } - return src + src_cur; + return src_cur; } // checks if the next 3 characters of the string pointer are the start of a @@ -96,7 +95,7 @@ strtointeger(const char *__restrict src, int base, if (base < 0 || base == 1 || base > 36) return {0, 0, EINVAL}; - src_cur = first_non_whitespace(src, src_len) - src; + src_cur = first_non_whitespace(src, src_len); char result_sign = '+'; if (src[src_cur] == '+' || src[src_cur] == '-') { @@ -119,7 +118,7 @@ strtointeger(const char *__restrict src, int base, ResultType const abs_max = (is_positive ? cpp::numeric_limits::max() : NEGATIVE_MAX); ResultType const abs_max_div_by_base = - static_cast(abs_max / base); + abs_max / static_cast(base); while (src_cur < src_len && isalnum(src[src_cur])) { int cur_digit = b36_char_to_int(src[src_cur]); @@ -141,17 +140,17 @@ strtointeger(const char *__restrict src, int base, result = abs_max; error_val = ERANGE; } else { - result = static_cast(result * base); + result = result * static_cast(base); } - if (result > abs_max - cur_digit) { + if (result > abs_max - static_cast(cur_digit)) { result = abs_max; error_val = ERANGE; } else { - result = static_cast(result + cur_digit); + result = result + static_cast(cur_digit); } } - ptrdiff_t str_len = is_number ? (src_cur) : 0; + ptrdiff_t str_len = is_number ? static_cast(src_cur) : 0; if (error_val == ERANGE) { if (is_positive || IS_UNSIGNED) diff --git a/lib/libcxx/libc/src/errno/libc_errno.h b/lib/libcxx/libc/src/errno/libc_errno.h deleted file mode 100644 index 44ee271484..0000000000 --- a/lib/libcxx/libc/src/errno/libc_errno.h +++ /dev/null @@ -1,47 +0,0 @@ -//===-- Implementation header for libc_errno --------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_LIBC_SRC_ERRNO_LIBC_ERRNO_H -#define LLVM_LIBC_SRC_ERRNO_LIBC_ERRNO_H - -#include "src/__support/macros/attributes.h" -#include "src/__support/macros/config.h" -#include "src/__support/macros/properties/architectures.h" - -#include "hdr/errno_macros.h" - -// This header is to be consumed by internal implementations, in which all of -// them should refer to `libc_errno` instead of using `errno` directly from -// header. - -// Unit and hermetic tests should: -// - #include "src/errno/libc_errno.h" -// - NOT #include -// - Only use `libc_errno` in the code -// - Depend on libc.src.errno.errno - -// Integration tests should: -// - NOT #include "src/errno/libc_errno.h" -// - #include -// - Use regular `errno` in the code -// - Still depend on libc.src.errno.errno - -namespace LIBC_NAMESPACE_DECL { - -extern "C" int *__llvm_libc_errno() noexcept; - -struct Errno { - void operator=(int); - operator int(); -}; - -extern Errno libc_errno; - -} // namespace LIBC_NAMESPACE_DECL - -#endif // LLVM_LIBC_SRC_ERRNO_LIBC_ERRNO_H