Skip to content

Use native rounding instructions for floor, ceil and round #714

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
a-eridani wants to merge 1 commit into
base: master
Choose a base branch
from
+117 −1
Open

Use native rounding instructions for floor, ceil and round #714

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions src/arch/helperadvsimd.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@

#define FULL_FP_ROUNDING
//@#define FULL_FP_ROUNDING
#define NATIVE_ROUND
//@#define NATIVE_ROUND
#define ACCURATE_SQRT
//@#define ACCURATE_SQRT

Expand Down Expand Up @@ -529,6 +531,16 @@ static INLINE vdouble vsel_vd_vo_vo_d_d_d(vopmask o0, vopmask o1, double d0, dou
static INLINE vdouble vrint_vd_vd(vdouble d) { return vrndnq_f64(d); }
static INLINE vfloat vrint_vf_vf(vfloat d) { return vrndnq_f32(d); }

static INLINE vdouble vfloor_vd_vd(vdouble v) { return vrndmq_f64(v); }
static INLINE vfloat vfloor_vf_vf(vfloat v) { return vrndmq_f32(v); }

static INLINE vdouble vceil_vd_vd(vdouble v) { return vrndpq_f64(v); }
static INLINE vfloat vceil_vf_vf(vfloat v) { return vrndpq_f32(v); }

static INLINE vdouble vround_vd_vd(vdouble v) { return vrndaq_f64(v); }
static INLINE vfloat vround_vf_vf(vfloat v) { return vrndaq_f32(v); }


/****************************************/
/* int <--> float conversions */
/****************************************/
Expand Down
8 changes: 8 additions & 0 deletions src/arch/helperavx2.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,14 @@ static INLINE vfloat vtruncate_vf_vf(vfloat vf) { return _mm256_round_ps(vf, _MM
static INLINE vdouble vcast_vd_vi(vint vi) { return _mm256_cvtepi32_pd(vi); }
static INLINE vint vcast_vi_i(int i) { return _mm_set1_epi32(i); }

static INLINE vdouble vfloor_vd_vd(vdouble v) { return _mm256_round_pd(v, _MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC); }
static INLINE vfloat vfloor_vf_vf(vfloat v) { return _mm256_round_ps(v, _MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC); }

static INLINE vdouble vceil_vd_vd(vdouble v) { return _mm256_round_pd(v, _MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC); }
static INLINE vfloat vceil_vf_vf(vfloat v) { return _mm256_round_ps(v, _MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC); }



static INLINE vmask vcastu_vm_vi(vint vi) {
return _mm256_slli_epi64(_mm256_cvtepi32_epi64(vi), 32);
}
Expand Down
7 changes: 7 additions & 0 deletions src/arch/helperavx2_128.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,13 @@ static INLINE vfloat vtruncate_vf_vf(vfloat vf) { return _mm_round_ps(vf, _MM_FR
static INLINE vdouble vcast_vd_vi(vint vi) { return _mm_cvtepi32_pd(vi); }
static INLINE vint vcast_vi_i(int i) { return _mm_set1_epi32(i); }

static INLINE vdouble vfloor_vd_vd(vdouble v) { return _mm_round_pd(v, _MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC); }
static INLINE vfloat vfloor_vf_vf(vfloat v) { return _mm_round_ps(v, _MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC); }

static INLINE vdouble vceil_vd_vd(vdouble v) { return _mm_round_pd(v, _MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC); }
static INLINE vfloat vceil_vf_vf(vfloat v) { return _mm_round_ps(v, _MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC); }


static INLINE vmask vcastu_vm_vi(vint vi) { return _mm_and_si128(_mm_shuffle_epi32(vi, 0x73), _mm_set_epi32(-1, 0, -1, 0)); }
static INLINE vint vcastu_vi_vm(vmask vi) { return _mm_shuffle_epi32(vi, 0x0d); }

Expand Down
6 changes: 6 additions & 0 deletions src/arch/helperavx512f.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -352,6 +352,12 @@ static INLINE vfloat vtruncate_vf_vf(vfloat vd) {
static INLINE vfloat vrint_vf_vf(vfloat vd) {
return _mm512_roundscale_ps(vd, _MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC);
}
static INLINE vdouble vfloor_vd_vd(vdouble v) { return _mm512_roundscale_pd(v, _MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC); }
static INLINE vfloat vfloor_vf_vf(vfloat v) { return _mm512_roundscale_ps(v, _MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC); }

static INLINE vdouble vceil_vd_vd(vdouble v) { return _mm512_roundscale_pd(v, _MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC); }
static INLINE vfloat vceil_vf_vf(vfloat v) { return _mm512_roundscale_ps(v, _MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC); }


static INLINE vfloat vadd_vf_vf_vf(vfloat x, vfloat y) { return _mm512_add_ps(x, y); }
static INLINE vfloat vsub_vf_vf_vf(vfloat x, vfloat y) { return _mm512_sub_ps(x, y); }
Expand Down
6 changes: 6 additions & 0 deletions src/arch/helperpower_128.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -430,6 +430,12 @@ static INLINE vdouble vrint_vd_vd(vdouble vd)
static INLINE vfloat vrint_vf_vf(vfloat vf)
{ return vec_rint(vf); }

static INLINE vdouble vfloor_vd_vd(vdouble v) { return vec_floor(v); }
static INLINE vfloat vfloor_vf_vf(vfloat v) { return vec_floor(v); }

static INLINE vdouble vceil_vd_vd(vdouble v) { return vec_ceil(v); }
static INLINE vfloat vceil_vf_vf(vfloat v) { return vec_ceil(v); }

/**********************************************
** Logical
**********************************************/
Expand Down
26 changes: 26 additions & 0 deletions src/arch/helperpurec_scalar.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,13 @@
#define TRUNC trunc
#define TRUNCF truncf

#define FLOOR floor
#define FLOORF floorf
#define CEIL ceil
#define CEILF ceilf
#define ROUND round
#define ROUNDF roundf

#else

#define SQRT __builtin_sqrt
Expand All @@ -38,6 +45,14 @@
#define TRUNC __builtin_trunc
#define TRUNCF __builtin_truncf

#define FLOOR __builtin_floor
#define FLOORF __builtin_floorf
#define CEIL __builtin_ceil
#define CEILF __builtin_ceilf
#define ROUND __builtin_round
#define ROUNDF __builtin_roundf


#endif

#if CONFIG == 1
Expand Down Expand Up @@ -91,6 +106,8 @@
#if defined(__aarch64__) || CONFIG == 3
#define FULL_FP_ROUNDING
//@#define FULL_FP_ROUNDING
#define NATIVE_ROUND
//@#define NATIVE_ROUND
#endif

#define DFTPRIORITY LOG2VECTLENDP
Expand Down Expand Up @@ -189,6 +206,15 @@ static INLINE vint vrint_vi_vd(vdouble d) { return (int32_t)RINT(d); }
static INLINE vdouble vrint_vd_vd(vdouble vd) { return RINT(vd); }
static INLINE vdouble vtruncate_vd_vd(vdouble vd) { return TRUNC(vd); }
static INLINE vint vtruncate_vi_vd(vdouble vd) { return (int32_t)TRUNC(vd); }

static INLINE vdouble vfloor_vd_vd(vdouble v) { return FLOOR(v); }
static INLINE vfloat vfloor_vf_vf(vfloat v) { return FLOORF(v); }

static INLINE vdouble vceil_vd_vd(vdouble v) { return CEIL(v); }
static INLINE vfloat vceil_vf_vf(vfloat v) { return CEILF(v); }

static INLINE vdouble vround_vd_vd(vdouble v) { return ROUND(v); }
static INLINE vfloat vround_vf_vf(vfloat v) { return ROUNDF(v); }
#else
static INLINE vint vrint_vi_vd(vdouble a) {
a += a > 0 ? 0.5 : -0.5;
Expand Down
9 changes: 9 additions & 0 deletions src/arch/helpers390x_128.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@

#define ACCURATE_SQRT
//@#define ACCURATE_SQRT
#define NATIVE_ROUND
//@#define NATIVE_ROUND
#define FULL_FP_ROUNDING
//@#define FULL_FP_ROUNDING

Expand Down Expand Up @@ -134,7 +136,14 @@ static INLINE vdouble vcast_vd_d(double d) { return (vdouble) { d, d }; }
static INLINE vdouble vcast_vd_vi(vint vi) { return (vdouble) { (double)vi[0], (double)vi[1] }; }
static INLINE vfloat vcast_vf_vi2(vint2 vi) { return (vfloat) { (float)vi[0], (float)vi[1], (float)vi[2], (float)vi[3] }; }
static INLINE vdouble vtruncate_vd_vd(vdouble vd) { return __builtin_s390_vfidb(vd, 4, 5); }

static INLINE vdouble vrint_vd_vd(vdouble vd) { return __builtin_s390_vfidb(vd, 4, 4); }
static INLINE vdouble vfloor_vd_vd(vdouble vd) { return __builtin_s390_vfidb(vd, 4, 7); }
static INLINE vfloat vfloor_vf_vf(vfloat vf) { return __builtin_s390_vfisb(vf, 4, 7); }
static INLINE vdouble vceil_vd_vd (vdouble vd) { return __builtin_s390_vfidb(vd, 4, 6); }
static INLINE vfloat vceil_vf_vf (vfloat vf) { return __builtin_s390_vfisb(vf, 4, 6); }
static INLINE vdouble vround_vd_vd(vdouble vd) { return __builtin_s390_vfidb(vd, 4, 1); }
static INLINE vfloat vround_vf_vf(vfloat vf) { return __builtin_s390_vfisb(vf, 4, 1); }

static INLINE vint vrint_vi_vd(vdouble vd) {
vd = vrint_vd_vd(vd);
Expand Down
12 changes: 12 additions & 0 deletions src/arch/helpersve.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,8 @@ static INLINE int vavailability_i(int name) { return 3; }

#define FULL_FP_ROUNDING
//@#define FULL_FP_ROUNDING
#define NATIVE_ROUND
//@#define NATIVE_ROUND
#define ACCURATE_SQRT
//@#define ACCURATE_SQRT

Expand Down Expand Up @@ -751,6 +753,16 @@ static INLINE vdouble vrint_vd_vd(vdouble vd) {
return svrintn_f64_x(ptrue, vd);
}

static INLINE vdouble vfloor_vd_vd(vdouble v) { return svrintm_f64_x(ptrue, v); }
static INLINE vfloat vfloor_vf_vf(vfloat v) { return svrintm_f32_x(ptrue, v); }

static INLINE vdouble vceil_vd_vd(vdouble v) { return svrintp_f64_x(ptrue, v); }
static INLINE vfloat vceil_vf_vf(vfloat v) { return svrintp_f32_x(ptrue, v); }

static INLINE vdouble vround_vd_vd(vdouble v) { return svrinta_f64_x(ptrue, v); }
static INLINE vfloat vround_vf_vf(vfloat v) { return svrinta_f32_x(ptrue, v); }


// FP math operations
static INLINE vdouble vadd_vd_vd_vd(vdouble x, vdouble y) {
return svadd_f64_x(ptrue, x, y);
Expand Down
14 changes: 13 additions & 1 deletion src/common/commonfuncs.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ static INLINE CONST vdouble vorsign_vd_vd_vd(vdouble x, vdouble y) {
}

static INLINE CONST vdouble vcopysign_vd_vd_vd(vdouble x, vdouble y) {
return vreinterpret_vd_vm(vxor_vm_vm_vm(vandnot_vm_vm_vm(vreinterpret_vm_vd(vcast_vd_d(-0.0)), vreinterpret_vm_vd(x)),
return vreinterpret_vd_vm(vxor_vm_vm_vm(vandnot_vm_vm_vm(vreinterpret_vm_vd(vcast_vd_d(-0.0)), vreinterpret_vm_vd(x)),
vand_vm_vm_vm (vreinterpret_vm_vd(vcast_vd_d(-0.0)), vreinterpret_vm_vd(y))));
}
#endif
Expand All @@ -242,27 +242,39 @@ static INLINE CONST vdouble vtruncate2_vd_vd(vdouble x) {
}

static INLINE CONST vdouble vfloor2_vd_vd(vdouble x) {
#ifdef FULL_FP_ROUNDING
return vfloor_vd_vd(x);
#else
vdouble fr = vsub_vd_vd_vd(x, vmul_vd_vd_vd(vcast_vd_d(INT64_C(1) << 31), vcast_vd_vi(vtruncate_vi_vd(vmul_vd_vd_vd(x, vcast_vd_d(1.0 / (INT64_C(1) << 31)))))));
fr = vsub_vd_vd_vd(fr, vcast_vd_vi(vtruncate_vi_vd(fr)));
fr = vsel_vd_vo_vd_vd(vlt_vo_vd_vd(fr, vcast_vd_d(0)), vadd_vd_vd_vd(fr, vcast_vd_d(1.0)), fr);
return vsel_vd_vo_vd_vd(vor_vo_vo_vo(visinf_vo_vd(x), vge_vo_vd_vd(vabs_vd_vd(x), vcast_vd_d(INT64_C(1) << 52))), x, vcopysign_vd_vd_vd(vsub_vd_vd_vd(x, fr), x));
#endif
}

static INLINE CONST vdouble vceil2_vd_vd(vdouble x) {
#ifdef FULL_FP_ROUNDING
return vceil_vd_vd(x);
#else
vdouble fr = vsub_vd_vd_vd(x, vmul_vd_vd_vd(vcast_vd_d(INT64_C(1) << 31), vcast_vd_vi(vtruncate_vi_vd(vmul_vd_vd_vd(x, vcast_vd_d(1.0 / (INT64_C(1) << 31)))))));
fr = vsub_vd_vd_vd(fr, vcast_vd_vi(vtruncate_vi_vd(fr)));
fr = vsel_vd_vo_vd_vd(vle_vo_vd_vd(fr, vcast_vd_d(0)), fr, vsub_vd_vd_vd(fr, vcast_vd_d(1.0)));
return vsel_vd_vo_vd_vd(vor_vo_vo_vo(visinf_vo_vd(x), vge_vo_vd_vd(vabs_vd_vd(x), vcast_vd_d(INT64_C(1) << 52))), x, vcopysign_vd_vd_vd(vsub_vd_vd_vd(x, fr), x));
#endif
}

static INLINE CONST vdouble vround2_vd_vd(vdouble d) {
#ifdef NATIVE_ROUND
return vround_vd_vd(d);
#else
vdouble x = vadd_vd_vd_vd(d, vcast_vd_d(0.5));
vdouble fr = vsub_vd_vd_vd(x, vmul_vd_vd_vd(vcast_vd_d(INT64_C(1) << 31), vcast_vd_vi(vtruncate_vi_vd(vmul_vd_vd_vd(x, vcast_vd_d(1.0 / (INT64_C(1) << 31)))))));
fr = vsub_vd_vd_vd(fr, vcast_vd_vi(vtruncate_vi_vd(fr)));
x = vsel_vd_vo_vd_vd(vand_vo_vo_vo(vle_vo_vd_vd(x, vcast_vd_d(0)), veq_vo_vd_vd(fr, vcast_vd_d(0))), vsub_vd_vd_vd(x, vcast_vd_d(1.0)), x);
fr = vsel_vd_vo_vd_vd(vlt_vo_vd_vd(fr, vcast_vd_d(0)), vadd_vd_vd_vd(fr, vcast_vd_d(1.0)), fr);
x = vsel_vd_vo_vd_vd(veq_vo_vd_vd(d, vcast_vd_d(0.49999999999999994449)), vcast_vd_d(0), x);
return vsel_vd_vo_vd_vd(vor_vo_vo_vo(visinf_vo_vd(d), vge_vo_vd_vd(vabs_vd_vd(d), vcast_vd_d(INT64_C(1) << 52))), d, vcopysign_vd_vd_vd(vsub_vd_vd_vd(x, fr), d));
#endif
}

static INLINE CONST vdouble vrint2_vd_vd(vdouble d) {
Expand Down
6 changes: 6 additions & 0 deletions src/common/keywords.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -386,6 +386,12 @@ vrev21_vf_vf
vreva2_vd_vd
vreva2_vf_vf
vrint_vd_vd
vfloor_vd_vd
vfloor_vf_vf
vceil_vd_vd
vceil_vf_vf
vround_vd_vd
vround_vf_vf
vrint2_vd_vd
vrint_vf_vf
vrint_vi2_vf
Expand Down
12 changes: 12 additions & 0 deletions src/libm/sleefsimdsp.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2163,24 +2163,36 @@ EXPORT CONST vfloat xtruncf(vfloat x) {
}

EXPORT CONST vfloat xfloorf(vfloat x) {
#ifdef FULL_FP_ROUNDING
return vfloor_vf_vf(x);
#else
vfloat fr = vsub_vf_vf_vf(x, vcast_vf_vi2(vtruncate_vi2_vf(x)));
fr = vsel_vf_vo_vf_vf(vlt_vo_vf_vf(fr, vcast_vf_f(0)), vadd_vf_vf_vf(fr, vcast_vf_f(1.0f)), fr);
return vsel_vf_vo_vf_vf(vor_vo_vo_vo(visinf_vo_vf(x), vge_vo_vf_vf(vabs_vf_vf(x), vcast_vf_f(INT64_C(1) << 23))), x, vcopysign_vf_vf_vf(vsub_vf_vf_vf(x, fr), x));
#endif
}

EXPORT CONST vfloat xceilf(vfloat x) {
#ifdef FULL_FP_ROUNDING
return vceil_vf_vf(x);
#else
vfloat fr = vsub_vf_vf_vf(x, vcast_vf_vi2(vtruncate_vi2_vf(x)));
fr = vsel_vf_vo_vf_vf(vle_vo_vf_vf(fr, vcast_vf_f(0)), fr, vsub_vf_vf_vf(fr, vcast_vf_f(1.0f)));
return vsel_vf_vo_vf_vf(vor_vo_vo_vo(visinf_vo_vf(x), vge_vo_vf_vf(vabs_vf_vf(x), vcast_vf_f(INT64_C(1) << 23))), x, vcopysign_vf_vf_vf(vsub_vf_vf_vf(x, fr), x));
#endif
}

EXPORT CONST vfloat xroundf(vfloat d) {
#ifdef NATIVE_ROUND
return vround_vf_vf(d);
#else
vfloat x = vadd_vf_vf_vf(d, vcast_vf_f(0.5f));
vfloat fr = vsub_vf_vf_vf(x, vcast_vf_vi2(vtruncate_vi2_vf(x)));
x = vsel_vf_vo_vf_vf(vand_vo_vo_vo(vle_vo_vf_vf(x, vcast_vf_f(0)), veq_vo_vf_vf(fr, vcast_vf_f(0))), vsub_vf_vf_vf(x, vcast_vf_f(1.0f)), x);
fr = vsel_vf_vo_vf_vf(vlt_vo_vf_vf(fr, vcast_vf_f(0)), vadd_vf_vf_vf(fr, vcast_vf_f(1.0f)), fr);
x = vsel_vf_vo_vf_vf(veq_vo_vf_vf(d, vcast_vf_f(0.4999999701976776123f)), vcast_vf_f(0), x);
return vsel_vf_vo_vf_vf(vor_vo_vo_vo(visinf_vo_vf(d), vge_vo_vf_vf(vabs_vf_vf(d), vcast_vf_f(INT64_C(1) << 23))), d, vcopysign_vf_vf_vf(vsub_vf_vf_vf(x, fr), d));
#endif
}

EXPORT CONST vfloat xrintf(vfloat d) {
Expand Down