[libclc] Only use software fma for r600 target (#179428)

Implement generic __clc_fma with __builtin_elementwise_fma for all
targets except for r600.

Add --spirv-ext=+SPV_KHR_fma flag to SPIR-V generation. SPIR-V target
supports @llvm.fma since SPV_KHR_fma was implemented in llvm-spirv
(https://github.com/KhronosGroup/SPIRV-LLVM-Translator/pull/3467) and
SPIR-V backend (8f8dfbf8c9).
This PR assumes SPIR-V consumer with modern hardware supports fma.

libclc/clc/include/clc/internal/math/clc_sw_fma.h

@@ -10,6 +10,7 @@
 #define __CLC_INTERNAL_MATH_CLC_SW_FMA_H__
 
 #define __CLC_FUNCTION __clc_sw_fma
+#define __CLC_FLOAT_ONLY
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
 
 #include <clc/math/gentype.inc>

libclc/clc/include/clc/math/math.h

@@ -11,7 +11,6 @@
 
 #include <clc/clc_as_type.h>
 #include <clc/clcfunc.h>
-#include <clc/math/clc_subnormal_config.h>
 
 #define SNAN 0x001
 #define QNAN 0x002
@@ -24,11 +23,8 @@
 #define PNOR 0x100
 #define PINF 0x200
 
-#if (defined __AMDGCN__ || defined __R600__) && !defined __HAS_FMAF__
+#ifdef __R600__
 #define __CLC_HAVE_HW_FMA32() (0)
-#elif defined(CLC_SPIRV)
-bool __attribute__((noinline)) __clc_runtime_has_hw_fma32(void);
-#define __CLC_HAVE_HW_FMA32() __clc_runtime_has_hw_fma32()
 #else
 #define __CLC_HAVE_HW_FMA32() (1)
 #endif
@@ -65,16 +61,6 @@ bool __attribute__((noinline)) __clc_runtime_has_hw_fma32(void);
 
 #define LOG_MAGIC_NUM_SP32 (1 + NUMEXPBITS_SP32 - EXPBIAS_SP32)
 
-_CLC_OVERLOAD _CLC_INLINE float __clc_flush_denormal_if_not_supported(float x) {
-  int ix = __clc_as_int(x);
-  if (!__clc_fp32_subnormals_supported() && ((ix & EXPBITS_SP32) == 0) &&
-      ((ix & MANTBITS_SP32) != 0)) {
-    ix &= SIGNBIT_SP32;
-    x = __clc_as_float(ix);
-  }
-  return x;
-}
-
 #ifdef cl_khr_fp64
 
 #define SIGNBIT_DP64 0x8000000000000000L

libclc/clc/lib/clspv/SOURCES

@@ -1,2 +1 @@
-math/clc_sw_fma.cl
 integer/clc_mul_hi.cl

libclc/clc/lib/clspv/math/clc_sw_fma.cl

@@ -1,274 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// This version is derived from the generic fma software implementation
-// (__clc_sw_fma), but avoids the use of ulong in favor of uint2. The logic has
-// been updated as appropriate.
-
-#include <clc/clc_as_type.h>
-#include <clc/float/definitions.h>
-#include <clc/integer/clc_abs.h>
-#include <clc/integer/clc_clz.h>
-#include <clc/integer/clc_hadd.h>
-#include <clc/integer/clc_mul_hi.h>
-#include <clc/integer/definitions.h>
-#include <clc/math/clc_mad.h>
-#include <clc/math/math.h>
-#include <clc/relational/clc_isinf.h>
-#include <clc/relational/clc_isnan.h>
-#include <clc/shared/clc_max.h>
-
-struct fp {
-  uint2 mantissa;
-  int exponent;
-  uint sign;
-};
-
-static uint2 u2_set(uint hi, uint lo) {
-  uint2 res;
-  res.lo = lo;
-  res.hi = hi;
-  return res;
-}
-
-static uint2 u2_set_u(uint val) { return u2_set(0, val); }
-
-static uint2 u2_mul(uint a, uint b) {
-  uint2 res;
-  res.hi = __clc_mul_hi(a, b);
-  res.lo = a * b;
-  return res;
-}
-
-static uint2 u2_sll(uint2 val, uint shift) {
-  if (shift == 0)
-    return val;
-  if (shift < 32) {
-    val.hi <<= shift;
-    val.hi |= val.lo >> (32 - shift);
-    val.lo <<= shift;
-  } else {
-    val.hi = val.lo << (shift - 32);
-    val.lo = 0;
-  }
-  return val;
-}
-
-static uint2 u2_srl(uint2 val, uint shift) {
-  if (shift == 0)
-    return val;
-  if (shift < 32) {
-    val.lo >>= shift;
-    val.lo |= val.hi << (32 - shift);
-    val.hi >>= shift;
-  } else {
-    val.lo = val.hi >> (shift - 32);
-    val.hi = 0;
-  }
-  return val;
-}
-
-static uint2 u2_or(uint2 a, uint b) {
-  a.lo |= b;
-  return a;
-}
-
-static uint2 u2_and(uint2 a, uint2 b) {
-  a.lo &= b.lo;
-  a.hi &= b.hi;
-  return a;
-}
-
-static uint2 u2_add(uint2 a, uint2 b) {
-  uint carry = (__clc_hadd(a.lo, b.lo) >> 31) & 0x1;
-  a.lo += b.lo;
-  a.hi += b.hi + carry;
-  return a;
-}
-
-static uint2 u2_add_u(uint2 a, uint b) { return u2_add(a, u2_set_u(b)); }
-
-static uint2 u2_inv(uint2 a) {
-  a.lo = ~a.lo;
-  a.hi = ~a.hi;
-  return u2_add_u(a, 1);
-}
-
-static uint u2_clz(uint2 a) {
-  uint leading_zeroes = __clc_clz(a.hi);
-  if (leading_zeroes == 32) {
-    leading_zeroes += __clc_clz(a.lo);
-  }
-  return leading_zeroes;
-}
-
-static bool u2_eq(uint2 a, uint2 b) { return a.lo == b.lo && a.hi == b.hi; }
-
-static bool u2_zero(uint2 a) { return u2_eq(a, u2_set_u(0)); }
-
-static bool u2_gt(uint2 a, uint2 b) {
-  return a.hi > b.hi || (a.hi == b.hi && a.lo > b.lo);
-}
-
-_CLC_DEF _CLC_OVERLOAD float __clc_sw_fma(float a, float b, float c) {
-  /* special cases */
-  if (__clc_isnan(a) || __clc_isnan(b) || __clc_isnan(c) || __clc_isinf(a) ||
-      __clc_isinf(b)) {
-    return __clc_mad(a, b, c);
-  }
-
-  /* If only c is inf, and both a,b are regular numbers, the result is c*/
-  if (__clc_isinf(c)) {
-    return c;
-  }
-
-  a = __clc_flush_denormal_if_not_supported(a);
-  b = __clc_flush_denormal_if_not_supported(b);
-  c = __clc_flush_denormal_if_not_supported(c);
-
-  if (a == 0.0f || b == 0.0f) {
-    return c;
-  }
-
-  if (c == 0) {
-    return a * b;
-  }
-
-  struct fp st_a, st_b, st_c;
-
-  st_a.exponent = a == .0f ? 0 : ((__clc_as_uint(a) & 0x7f800000) >> 23) - 127;
-  st_b.exponent = b == .0f ? 0 : ((__clc_as_uint(b) & 0x7f800000) >> 23) - 127;
-  st_c.exponent = c == .0f ? 0 : ((__clc_as_uint(c) & 0x7f800000) >> 23) - 127;
-
-  st_a.mantissa =
-      u2_set_u(a == .0f ? 0 : (__clc_as_uint(a) & 0x7fffff) | 0x800000);
-  st_b.mantissa =
-      u2_set_u(b == .0f ? 0 : (__clc_as_uint(b) & 0x7fffff) | 0x800000);
-  st_c.mantissa =
-      u2_set_u(c == .0f ? 0 : (__clc_as_uint(c) & 0x7fffff) | 0x800000);
-
-  st_a.sign = __clc_as_uint(a) & 0x80000000;
-  st_b.sign = __clc_as_uint(b) & 0x80000000;
-  st_c.sign = __clc_as_uint(c) & 0x80000000;
-
-  // Multiplication.
-  // Move the product to the highest bits to maximize precision
-  // mantissa is 24 bits => product is 48 bits, 2bits non-fraction.
-  // Add one bit for future addition overflow,
-  // add another bit to detect subtraction underflow
-  struct fp st_mul;
-  st_mul.sign = st_a.sign ^ st_b.sign;
-  st_mul.mantissa = u2_sll(u2_mul(st_a.mantissa.lo, st_b.mantissa.lo), 14);
-  st_mul.exponent =
-      !u2_zero(st_mul.mantissa) ? st_a.exponent + st_b.exponent : 0;
-
-  // FIXME: Detecting a == 0 || b == 0 above crashed GCN isel
-  if (st_mul.exponent == 0 && u2_zero(st_mul.mantissa))
-    return c;
-
-// Mantissa is 23 fractional bits, shift it the same way as product mantissa
-#define C_ADJUST 37ul
-
-  // both exponents are bias adjusted
-  int exp_diff = st_mul.exponent - st_c.exponent;
-
-  st_c.mantissa = u2_sll(st_c.mantissa, C_ADJUST);
-  uint2 cutoff_bits = u2_set_u(0);
-  uint2 cutoff_mask = u2_add(u2_sll(u2_set_u(1), __clc_abs(exp_diff)),
-                             u2_set(0xffffffff, 0xffffffff));
-  if (exp_diff > 0) {
-    cutoff_bits =
-        exp_diff >= 64 ? st_c.mantissa : u2_and(st_c.mantissa, cutoff_mask);
-    st_c.mantissa =
-        exp_diff >= 64 ? u2_set_u(0) : u2_srl(st_c.mantissa, exp_diff);
-  } else {
-    cutoff_bits = -exp_diff >= 64 ? st_mul.mantissa
-                                  : u2_and(st_mul.mantissa, cutoff_mask);
-    st_mul.mantissa =
-        -exp_diff >= 64 ? u2_set_u(0) : u2_srl(st_mul.mantissa, -exp_diff);
-  }
-
-  struct fp st_fma;
-  st_fma.sign = st_mul.sign;
-  st_fma.exponent = __clc_max(st_mul.exponent, st_c.exponent);
-  if (st_c.sign == st_mul.sign) {
-    st_fma.mantissa = u2_add(st_mul.mantissa, st_c.mantissa);
-  } else {
-    // cutoff bits borrow one
-    st_fma.mantissa =
-        u2_add(u2_add(st_mul.mantissa, u2_inv(st_c.mantissa)),
-               (!u2_zero(cutoff_bits) && (st_mul.exponent > st_c.exponent)
-                    ? u2_set(0xffffffff, 0xffffffff)
-                    : u2_set_u(0)));
-  }
-
-  // underflow: st_c.sign != st_mul.sign, and magnitude switches the sign
-  if (u2_gt(st_fma.mantissa, u2_set(0x7fffffff, 0xffffffff))) {
-    st_fma.mantissa = u2_inv(st_fma.mantissa);
-    st_fma.sign = st_mul.sign ^ 0x80000000;
-  }
-
-  // detect overflow/underflow
-  int overflow_bits = 3 - u2_clz(st_fma.mantissa);
-
-  // adjust exponent
-  st_fma.exponent += overflow_bits;
-
-  // handle underflow
-  if (overflow_bits < 0) {
-    st_fma.mantissa = u2_sll(st_fma.mantissa, -overflow_bits);
-    overflow_bits = 0;
-  }
-
-  // rounding
-  uint2 trunc_mask = u2_add(u2_sll(u2_set_u(1), C_ADJUST + overflow_bits),
-                            u2_set(0xffffffff, 0xffffffff));
-  uint2 trunc_bits =
-      u2_or(u2_and(st_fma.mantissa, trunc_mask), !u2_zero(cutoff_bits));
-  uint2 last_bit =
-      u2_and(st_fma.mantissa, u2_sll(u2_set_u(1), C_ADJUST + overflow_bits));
-  uint2 grs_bits = u2_sll(u2_set_u(4), C_ADJUST - 3 + overflow_bits);
-
-  // round to nearest even
-  if (u2_gt(trunc_bits, grs_bits) ||
-      (u2_eq(trunc_bits, grs_bits) && !u2_zero(last_bit))) {
-    st_fma.mantissa =
-        u2_add(st_fma.mantissa, u2_sll(u2_set_u(1), C_ADJUST + overflow_bits));
-  }
-
-  // Shift mantissa back to bit 23
-  st_fma.mantissa = u2_srl(st_fma.mantissa, C_ADJUST + overflow_bits);
-
-  // Detect rounding overflow
-  if (u2_gt(st_fma.mantissa, u2_set_u(0xffffff))) {
-    ++st_fma.exponent;
-    st_fma.mantissa = u2_srl(st_fma.mantissa, 1);
-  }
-
-  if (u2_zero(st_fma.mantissa)) {
-    return 0.0f;
-  }
-
-  // Flating point range limit
-  if (st_fma.exponent > 127) {
-    return __clc_as_float(__clc_as_uint(INFINITY) | st_fma.sign);
-  }
-
-  // Flush denormals
-  if (st_fma.exponent <= -127) {
-    return __clc_as_float(st_fma.sign);
-  }
-
-  return __clc_as_float(st_fma.sign | ((st_fma.exponent + 127) << 23) |
-                        ((uint)st_fma.mantissa.lo & 0x7fffff));
-}
-
-#define __CLC_FLOAT_ONLY
-#define __CLC_FUNCTION __clc_sw_fma
-#define __CLC_BODY <clc/shared/ternary_def_scalarize.inc>
-#include <clc/math/gentype.inc>

libclc/clc/lib/generic/SOURCES

@@ -143,7 +143,6 @@ math/clc_sincos_helpers.cl
 math/clc_sinh.cl
 math/clc_sinpi.cl
 math/clc_sqrt.cl
-math/clc_sw_fma.cl
 math/clc_tables.cl
 math/clc_tan.cl
 math/clc_tanh.cl

libclc/clc/lib/generic/math/clc_fma.cl

@@ -7,8 +7,11 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/internal/clc.h>
-#include <clc/internal/math/clc_sw_fma.h>
+#include <clc/math/clc_fma.h>
 #include <clc/math/math.h>
 
-#define __CLC_BODY <clc_fma.inc>
+#define __CLC_FUNCTION __clc_fma
+#define __CLC_IMPL_FUNCTION(x) __builtin_elementwise_fma
+#define __CLC_BODY <clc/shared/ternary_def.inc>
+
 #include <clc/math/gentype.inc>

libclc/clc/lib/r600/SOURCES

@@ -1,2 +1,4 @@
+math/clc_fma.cl
 math/clc_native_rsqrt.cl
 math/clc_rsqrt.cl
+math/clc_sw_fma.cl

libclc/clc/lib/r600/math/clc_fma.cl

@@ -6,11 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <clc/internal/clc.h>
 #include <clc/internal/math/clc_sw_fma.h>
+#include <clc/math/math.h>
 
-#define __CLC_FLOAT_ONLY
-#define __CLC_FUNCTION fma
-#define __CLC_IMPL_FUNCTION(x) __clc_sw_fma
-#define __CLC_BODY <clc/shared/ternary_def.inc>
-
+#define __CLC_BODY <clc_fma.inc>
 #include <clc/math/gentype.inc>

libclc/clc/lib/r600/math/clc_fma.inc

@@ -9,8 +9,8 @@
 _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_fma(__CLC_GENTYPE a, __CLC_GENTYPE b,
                                                __CLC_GENTYPE c) {
 #if __CLC_FPSIZE == 32
-  if (!__CLC_HAVE_HW_FMA32())
-    return __clc_sw_fma(a, b, c);
-#endif
+  return __clc_sw_fma(a, b, c);
+#else
   return __builtin_elementwise_fma(a, b, c);
+#endif
 }

libclc/clc/lib/r600/math/clc_sw_fma.cl

@@ -18,6 +18,15 @@
 #include <clc/relational/clc_isnan.h>
 #include <clc/shared/clc_max.h>
 
+static _CLC_INLINE float __clc_flush_denormal(float x) {
+  int ix = __clc_as_int(x);
+  if (((ix & EXPBITS_SP32) == 0) && ((ix & MANTBITS_SP32) != 0)) {
+    ix &= SIGNBIT_SP32;
+    x = __clc_as_float(ix);
+  }
+  return x;
+}
+
 struct fp {
   ulong mantissa;
   int exponent;
@@ -36,9 +45,9 @@ _CLC_DEF _CLC_OVERLOAD float __clc_sw_fma(float a, float b, float c) {
     return c;
   }
 
-  a = __clc_flush_denormal_if_not_supported(a);
-  b = __clc_flush_denormal_if_not_supported(b);
-  c = __clc_flush_denormal_if_not_supported(c);
+  a = __clc_flush_denormal(a);
+  b = __clc_flush_denormal(b);
+  c = __clc_flush_denormal(c);
 
   if (c == 0) {
     return a * b;

libclc/clc/lib/spirv/SOURCES

@@ -1,3 +1,2 @@
 math/clc_fmax.cl
 math/clc_fmin.cl
-math/clc_runtime_has_hw_fma32.cl

libclc/clc/lib/spirv/math/clc_runtime_has_hw_fma32.cl

@@ -1,9 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-bool __clc_runtime_has_hw_fma32() { return false; }

libclc/cmake/modules/AddLibclc.cmake

@@ -394,12 +394,15 @@ function(add_libclc_builtin_set)
     set( libclc_builtins_lib ${library_dir}/${LIBCLC_OUTPUT_FILENAME}.spv )
     if ( LIBCLC_USE_SPIRV_BACKEND )
       add_custom_command( OUTPUT ${libclc_builtins_lib}
-        COMMAND ${clang_exe} -c --target=${ARG_TRIPLE} -x ir -o ${libclc_builtins_lib} ${builtins_link_lib}
+        COMMAND ${clang_exe} -c --target=${ARG_TRIPLE}
+                -mllvm --spirv-ext=+SPV_KHR_fma
+                -x ir -o ${libclc_builtins_lib} ${builtins_link_lib}
         DEPENDS ${clang_target} ${builtins_link_lib} ${builtins_link_lib_tgt}
       )
     else()
       add_custom_command( OUTPUT ${libclc_builtins_lib}
-        COMMAND ${llvm-spirv_exe} ${spvflags} -o ${libclc_builtins_lib} ${builtins_link_lib}
+        COMMAND ${llvm-spirv_exe} ${spvflags} --spirv-ext=+SPV_KHR_fma
+                -o ${libclc_builtins_lib} ${builtins_link_lib}
         DEPENDS ${llvm-spirv_target} ${builtins_link_lib} ${builtins_link_lib_tgt}
       )
     endif()

libclc/opencl/lib/clspv/SOURCES

@@ -2,7 +2,6 @@ conversion/convert_float2float.cl
 conversion/convert_float2int.cl
 conversion/convert_int2float.cl
 conversion/convert_integer.cl
-math/fma.cl
 shared/vstore_half.cl
 subnormal_config.cl
 ../generic/geometric/distance.cl

libclc/opencl/lib/clspv/math/fma.cl

@@ -1,16 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include <clc/internal/math/clc_sw_fma.h>
-
-#define __CLC_FLOAT_ONLY
-#define __CLC_FUNCTION fma
-#define __CLC_IMPL_FUNCTION(x) __clc_sw_fma
-#define __CLC_BODY <clc/shared/ternary_def.inc>
-
-#include <clc/math/gentype.inc>

libclc/opencl/lib/spirv/SOURCES

@@ -38,7 +38,6 @@ subnormal_config.cl
 ../generic/math/expm1.cl
 ../generic/math/exp2.cl
 ../generic/math/exp10.cl
-math/fma.cl
 ../generic/math/fmod.cl
 ../generic/math/fract.cl
 ../generic/math/frexp.cl