Fix cmath to handle some edge cases in complex number calculation (RustPython#3987)

* Fix: handle edge case for complex comparison, when both is both NaN+NaNj
* Fix: handle f64 overflow complex __abs__

Author: jirheee

Lib/test/test_cmath.py

@@ -173,8 +173,6 @@ def test_infinity_and_nan_constants(self):
         self.assertEqual(repr(cmath.nan), "nan")
         self.assertEqual(repr(cmath.nanj), "nanj")
 
-    # TODO: RUSTPYTHON see TODO in cmath_log.
-    @unittest.expectedFailure
     def test_user_object(self):
         # Test automatic calling of __complex__ and __float__ by cmath
         # functions
@@ -536,8 +534,6 @@ def test_abs(self):
         self.assertEqual(abs(complex(INF, NAN)), INF)
         self.assertTrue(math.isnan(abs(complex(NAN, NAN))))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     @requires_IEEE_754
     def test_abs_overflows(self):
         # result overflows

vm/src/builtins/complex.rs

@@ -228,9 +228,15 @@ impl PyComplex {
     }
 
     #[pymethod(magic)]
-    fn abs(&self) -> f64 {
+    fn abs(&self, vm: &VirtualMachine) -> PyResult<f64> {
         let Complex64 { im, re } = self.value;
-        re.hypot(im)
+        let is_finite = im.is_finite() && re.is_finite();
+        let abs_result = re.hypot(im);
+        if is_finite && abs_result.is_infinite() {
+            Err(vm.new_overflow_error("absolute value too large".to_string()))
+        } else {
+            Ok(abs_result)
+        }
     }
 
     #[inline]
@@ -402,7 +408,15 @@ impl Comparable for PyComplex {
     ) -> PyResult<PyComparisonValue> {
         op.eq_only(|| {
             let result = if let Some(other) = other.payload_if_subclass::<PyComplex>(vm) {
-                zelf.value == other.value
+                if zelf.value.re.is_nan()
+                    && zelf.value.im.is_nan()
+                    && other.value.re.is_nan()
+                    && other.value.im.is_nan()
+                {
+                    true
+                } else {
+                    zelf.value == other.value
+                }
             } else {
                 match float::to_op_float(other, vm) {
                     Ok(Some(other)) => zelf.value == other.into(),