libm: Fix carry condition in floor/ceil

The floor and ceil implementations work 32-bits at a time, so they have
to special case a carry from the lower 32-bit value into the higher
value. We need to be explicit and use unsigned compares to detect when
the overflow occurs.

Closes: picolibc#480

Signed-off-by: Keith Packard <[email protected]>

Author: keith-packard

newlib/libm/math/s_ceil.c

@@ -63,7 +63,7 @@ ceil64(__float64 x)
                 i0 += 1;
             else {
                 j = i1 + (1 << (52 - j0));
-                if (j < i1)
+                if ((__uint32_t) j < (__uint32_t) i1)
                     i0 += 1; /* got a carry */
                 i1 = j;
             }

newlib/libm/math/s_floor.c

@@ -104,7 +104,7 @@ floor64(__float64 x)
                 i0 += 1;
             else {
                 j = i1 + (1 << (52 - j0));
-                if (j < i1)
+                if ((__uint32_t) j < (__uint32_t) i1)
                     i0 += 1; /* got a carry */
                 i1 = j;
             }

newlib/libm/test/ceil_vec.c

@@ -300,6 +300,7 @@
 {64, 0,123,__LINE__, 0x403b0000, 0x00000000, 0x403a1999, 0x99999996},	/* 27=f(26.1)*/
 {64, 0,123,__LINE__, 0x403c0000, 0x00000000, 0x403bcccc, 0xccccccc9},	/* 28=f(27.8)*/
 {64, 0,123,__LINE__, 0x403e0000, 0x00000000, 0x403d7fff, 0xfffffffc},	/* 30=f(29.5)*/
+{64, 0,123,__LINE__, 0x41500000, 0x00000000, 0x414fffff, 0xc0000000},	/* 0x1p+22=f(0x1.fffffcp+21)*/
 {64, 0,123,__LINE__, 0x7ff80000, 0x00000000, 0x7ff80000, 0x00000000},   /* qnan=f(qnan)*/
 {64, 0,123,__LINE__, 0x7ff80000, 0x00000000, 0x7ff40000, 0x00000000},   /* qnan=f(snan)*/
 {64, 0,123,__LINE__, 0x7ff00000, 0x00000000, 0x7ff00000, 0x00000000},   /* +inf=f(+inf)*/

newlib/libm/test/floor_vec.c

@@ -300,6 +300,7 @@
 {64, 0,123,__LINE__, 0x403a0000, 0x00000000, 0x403a1999, 0x99999996},	/* 26=f(26.1)*/
 {64, 0,123,__LINE__, 0x403b0000, 0x00000000, 0x403bcccc, 0xccccccc9},	/* 27=f(27.8)*/
 {64, 0,123,__LINE__, 0x403d0000, 0x00000000, 0x403d7fff, 0xfffffffc},	/* 29=f(29.5)*/
+{64, 0,123,__LINE__, 0xc1500000, 0x00000000, 0xc14fffff, 0xc0000000},	/* -0x1p+22=f(-0x1.fffffcp+21)*/
 {64, 0,123,__LINE__, 0x7ff80000, 0x00000000, 0x7ff80000, 0x00000000},   /* qnan=f(qnan)*/
 {64, 0,123,__LINE__, 0x7ff80000, 0x00000000, 0x7ff40000, 0x00000000},   /* qnan=f(snan)*/
 {64, 0,123,__LINE__, 0x7ff00000, 0x00000000, 0x7ff00000, 0x00000000},   /* +inf=f(+inf)*/