Optimize add and sub

include/intx/int128.hpp

@@ -69,28 +69,52 @@ struct uint<128>
 using uint128 = uint<128>;
 
 
+template <unsigned N>
+struct uint_with_carry
+{
+    uint<N> value;
+    bool carry;
+};
+
+
 /// Linear arithmetic operators.
 /// @{
 
+constexpr uint_with_carry<128> add_with_carry(uint128 a, uint128 b) noexcept
+{
+    const auto lo = a.lo + b.lo;
+    const auto lo_carry = lo < a.lo;
+    const auto t = a.hi + b.hi;
+    const auto carry1 = t < a.hi;
+    const auto hi = t + lo_carry;
+    const auto carry2 = hi < t;
+    return {{hi, lo}, carry1 || carry2};
+}
+
 constexpr uint128 operator+(uint128 x, uint128 y) noexcept
 {
-    const auto lo = x.lo + y.lo;
-    const auto carry = x.lo > lo;
-    const auto hi = x.hi + y.hi + carry;
-    return {hi, lo};
+    return add_with_carry(x, y).value;
 }
 
 constexpr uint128 operator+(uint128 x) noexcept
 {
     return x;
 }
 
+constexpr uint_with_carry<128> sub_with_borrow(uint128 a, uint128 b) noexcept
+{
+    const auto lo = a.lo - b.lo;
+    const auto lo_borrow = lo > a.lo;
+    const auto t = a.hi - b.hi;
+    const auto borrow1 = t > a.hi;
+    const auto hi = t - lo_borrow;
+    const auto borrow2 = hi > t;
+    return {{hi, lo}, borrow1 || borrow2};
+}
+
 constexpr uint128 operator-(uint128 x, uint128 y) noexcept
 {
-    const auto lo = x.lo - y.lo;
-    const auto borrow = x.lo < lo;
-    const auto hi = x.hi - y.hi - borrow;
-    return {hi, lo};
+    return sub_with_borrow(x, y).value;
 }
 
 constexpr uint128 operator-(uint128 x) noexcept

include/intx/intx.hpp

@@ -471,21 +471,6 @@ inline uint<N> shl_loop(const uint<N>& x, unsigned shift)
 }
 
 
-template <unsigned N>
-struct uint_with_carry
-{
-    uint<N> value;
-    bool carry;
-};
-
-constexpr uint_with_carry<128> add_with_carry(uint128 a, uint128 b) noexcept
-{
-    // FIXME: Rename add_with_carry() to add_overflow().
-    const auto s = a + b;
-    const auto k = s < a;
-    return {s, k};
-}
-
 template <unsigned N>
 constexpr uint_with_carry<N> add_with_carry(const uint<N>& a, const uint<N>& b) noexcept
 {
@@ -495,6 +480,16 @@ constexpr uint_with_carry<N> add_with_carry(const uint<N>& a, const uint<N>& b)
     return {{hi.value, lo.value}, tt.carry || hi.carry};
 }
 
+
+template <unsigned N>
+constexpr uint_with_carry<N> sub_with_borrow(const uint<N>& a, const uint<N>& b) noexcept
+{
+    const auto lo = sub_with_borrow(a.lo, b.lo);
+    const auto tt = sub_with_borrow(a.hi, b.hi);
+    const auto hi = sub_with_borrow(tt.value, typename uint<N>::half_type{lo.carry});
+    return {{hi.value, lo.value}, tt.carry || hi.carry};
+}
+
 template <unsigned N>
 inline uint<N> add_loop(const uint<N>& a, const uint<N>& b) noexcept
 {
@@ -533,7 +528,7 @@ constexpr uint<N> operator-(const uint<N>& x) noexcept
 template <unsigned N>
 constexpr uint<N> operator-(const uint<N>& x, const uint<N>& y) noexcept
 {
-    return x + -y;
+    return sub_with_borrow(x, y).value;
 }
 
 template <unsigned N, typename T,

test/benchmarks/benchmarks.cpp

@@ -286,10 +286,10 @@ inline auto public_mul(const uint512& x, const uint512& y) noexcept
     return x * y;
 }
 
-BENCHMARK_TEMPLATE(binary_op512, inline_add);
 BENCHMARK_TEMPLATE(binary_op512, add);
-BENCHMARK_TEMPLATE(binary_op512, inline_sub);
+BENCHMARK_TEMPLATE(binary_op512, inline_add);
 BENCHMARK_TEMPLATE(binary_op512, sub);
+BENCHMARK_TEMPLATE(binary_op512, inline_sub);
 BENCHMARK_TEMPLATE(binary_op512, mul);
 BENCHMARK_TEMPLATE(binary_op512, public_mul);
 BENCHMARK_TEMPLATE(binary_op512, gmp::mul);

test/experimental/add.cpp

@@ -69,21 +69,37 @@ uint192 add_waterfall(
 
 uint256 add_recursive(const uint256& a, const uint256& b) noexcept
 {
-    const auto ll = a.lo.lo + b.lo.lo;
-    const auto l_carry = ll < a.lo.lo;
-    const auto lh = a.lo.hi + b.lo.hi + l_carry;
-
-    const bool lo_carry = (lh < a.lo.hi) | ((lh == a.lo.hi) & l_carry);
-
-    const auto tl = a.hi.lo + b.hi.lo;
-    const auto t_carry = tl < a.hi.lo;
-    const auto th = a.hi.hi + b.hi.hi + t_carry;
-
-    const auto hl = tl + lo_carry;
-    const auto hl_carry = hl < tl;
-    const auto hh = th + hl_carry;
+    uint128 lo;
+    bool lo_carry;
+    {
+        const auto l = a.lo.lo + b.lo.lo;
+        const auto l_carry = l < a.lo.lo;
+        const auto t = a.lo.hi + b.lo.hi;
+        const auto carry1 = t < a.lo.hi;
+        const auto h = t + l_carry;
+        const auto carry2 = h < t;
+        lo = uint128{h, l};
+        lo_carry = carry1 || carry2;
+    }
+
+    uint128 tt;
+    {
+        const auto l = a.hi.lo + b.hi.lo;
+        const auto l_carry = l < a.hi.lo;
+        const auto t = a.hi.hi + b.hi.hi;
+        const auto h = t + l_carry;
+        tt = uint128{h, l};
+    }
 
-    return {{hh, hl}, {lh, ll}};
+    uint128 hi;
+    {
+        const auto l = tt.lo + lo_carry;
+        const auto l_carry = l < tt.lo;
+        const auto h = tt.hi + l_carry;
+        hi = uint128{h, l};
+    }
+
+    return {hi, lo};
 }
 
 uint256 add_waterflow(const uint256& a, const uint256& b) noexcept