refactor double-try-except and fix bug

MarcoGorelli · MarcoGorelli · commit 79ce542ac557 · 2023-01-05T15:34:54.000Z
diff --git a/doc/source/whatsnew/v2.0.0.rst b/doc/source/whatsnew/v2.0.0.rst
@@ -823,7 +823,8 @@ Datetimelike
 - Bug in :func:`to_datetime` was throwing ``ValueError`` when parsing dates with ISO8601 format where some values were not zero-padded (:issue:`21422`)
 - Bug in :func:`to_datetime` was giving incorrect results when using ``format='%Y%m%d'`` and ``errors='ignore'`` (:issue:`26493`)
 - Bug in :func:`to_datetime` was failing to parse date strings ``'today'`` and ``'now'`` if ``format`` was not ISO8601 (:issue:`50359`)
-- Bug in :func:`Timestamp.utctimetuple` raising a ``TypeError`` (:issue:`32174`)
+- Bug in :func:`to_datetime` was raising ``ValueError`` when parsing mixed-offset :class:`Timestamp` with ``errors='ignore'`` (:issue:`50585`)
+-
 
 Timedelta
 ^^^^^^^^^
diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx
@@ -30,7 +30,6 @@ from pandas._libs.tslibs.np_datetime cimport (
     NPY_DATETIMEUNIT,
     NPY_FR_ns,
     check_dts_bounds,
-    get_datetime64_value,
     npy_datetimestruct,
     npy_datetimestruct_to_datetime,
     pandas_datetime_to_datetimestruct,
@@ -505,144 +504,136 @@ cpdef array_to_datetime(
     result = np.empty(n, dtype="M8[ns]")
     iresult = result.view("i8")
 
-    try:
-        for i in range(n):
-            val = values[i]
+    for i in range(n):
+        val = values[i]
 
-            try:
-                if checknull_with_nat_and_na(val):
-                    iresult[i] = NPY_NAT
+        try:
+            if checknull_with_nat_and_na(val):
+                iresult[i] = NPY_NAT
 
-                elif PyDateTime_Check(val):
-                    if val.tzinfo is not None:
-                        found_tz = True
-                    else:
-                        found_naive = True
-                    tz_out = convert_timezone(
-                        val.tzinfo,
-                        tz_out,
-                        found_naive,
-                        found_tz,
-                        utc_convert,
-                    )
-                    result[i] = parse_pydatetime(val, &dts, utc_convert)
+            elif PyDateTime_Check(val):
+                if val.tzinfo is not None:
+                    found_tz = True
+                else:
+                    found_naive = True
+                tz_out = convert_timezone(
+                    val.tzinfo,
+                    tz_out,
+                    found_naive,
+                    found_tz,
+                    utc_convert,
+                )
+                result[i] = parse_pydatetime(val, &dts, utc_convert)
+
+            elif PyDate_Check(val):
+                iresult[i] = pydate_to_dt64(val, &dts)
+                check_dts_bounds(&dts)
 
-                elif PyDate_Check(val):
-                    iresult[i] = pydate_to_dt64(val, &dts)
-                    check_dts_bounds(&dts)
+            elif is_datetime64_object(val):
+                iresult[i] = get_datetime64_nanos(val, NPY_FR_ns)
 
-                elif is_datetime64_object(val):
-                    iresult[i] = get_datetime64_nanos(val, NPY_FR_ns)
+            elif is_integer_object(val) or is_float_object(val):
+                # these must be ns unit by-definition
 
-                elif is_integer_object(val) or is_float_object(val):
-                    # these must be ns unit by-definition
+                if val != val or val == NPY_NAT:
+                    iresult[i] = NPY_NAT
+                elif is_raise or is_ignore:
+                    iresult[i] = val
+                else:
+                    # coerce
+                    # we now need to parse this as if unit='ns'
+                    # we can ONLY accept integers at this point
+                    # if we have previously (or in future accept
+                    # datetimes/strings, then we must coerce)
+                    iresult[i] = cast_from_unit(val, "ns")
 
-                    if val != val or val == NPY_NAT:
-                        iresult[i] = NPY_NAT
-                    elif is_raise or is_ignore:
-                        iresult[i] = val
-                    else:
-                        # coerce
-                        # we now need to parse this as if unit='ns'
-                        # we can ONLY accept integers at this point
-                        # if we have previously (or in future accept
-                        # datetimes/strings, then we must coerce)
-                        try:
-                            iresult[i] = cast_from_unit(val, "ns")
-                        except OverflowError:
-                            iresult[i] = NPY_NAT
+            elif isinstance(val, str):
+                # string
+                if type(val) is not str:
+                    # GH#32264 np.str_ object
+                    val = str(val)
 
-                elif isinstance(val, str):
-                    # string
-                    if type(val) is not str:
-                        # GH#32264 np.str_ object
-                        val = str(val)
+                if len(val) == 0 or val in nat_strings:
+                    iresult[i] = NPY_NAT
+                    continue
 
-                    if len(val) == 0 or val in nat_strings:
-                        iresult[i] = NPY_NAT
+                string_to_dts_failed = string_to_dts(
+                    val, &dts, &out_bestunit, &out_local,
+                    &out_tzoffset, False, None, False
+                )
+                if string_to_dts_failed:
+                    # An error at this point is a _parsing_ error
+                    # specifically _not_ OutOfBoundsDatetime
+                    if parse_today_now(val, &iresult[i], utc):
                         continue
 
-                    string_to_dts_failed = string_to_dts(
-                        val, &dts, &out_bestunit, &out_local,
-                        &out_tzoffset, False, None, False
-                    )
-                    if string_to_dts_failed:
-                        # An error at this point is a _parsing_ error
-                        # specifically _not_ OutOfBoundsDatetime
-                        if parse_today_now(val, &iresult[i], utc):
+                    try:
+                        py_dt = parse_datetime_string(val,
+                                                      dayfirst=dayfirst,
+                                                      yearfirst=yearfirst)
+                        # If the dateutil parser returned tzinfo, capture it
+                        # to check if all arguments have the same tzinfo
+                        tz = py_dt.utcoffset()
+
+                    except (ValueError, OverflowError):
+                        if is_coerce:
+                            iresult[i] = NPY_NAT
                             continue
-
-                        try:
-                            py_dt = parse_datetime_string(val,
-                                                          dayfirst=dayfirst,
-                                                          yearfirst=yearfirst)
-                            # If the dateutil parser returned tzinfo, capture it
-                            # to check if all arguments have the same tzinfo
-                            tz = py_dt.utcoffset()
-
-                        except (ValueError, OverflowError):
-                            if is_coerce:
-                                iresult[i] = NPY_NAT
-                                continue
-                            raise TypeError(
-                                f"invalid string coercion to datetime "
-                                f"for \"{val}\", at position {i}"
-                            )
-
-                        if tz is not None:
-                            seen_datetime_offset = True
-                            # dateutil timezone objects cannot be hashed, so
-                            # store the UTC offsets in seconds instead
-                            out_tzoffset_vals.add(tz.total_seconds())
-                        else:
-                            # Add a marker for naive string, to track if we are
-                            # parsing mixed naive and aware strings
-                            out_tzoffset_vals.add("naive")
-
-                        _ts = convert_datetime_to_tsobject(py_dt, None)
-                        iresult[i] = _ts.value
-                    if not string_to_dts_failed:
-                        # No error reported by string_to_dts, pick back up
-                        # where we left off
-                        value = npy_datetimestruct_to_datetime(NPY_FR_ns, &dts)
-                        if out_local == 1:
-                            seen_datetime_offset = True
-                            # Store the out_tzoffset in seconds
-                            # since we store the total_seconds of
-                            # dateutil.tz.tzoffset objects
-                            out_tzoffset_vals.add(out_tzoffset * 60.)
-                            tz = timezone(timedelta(minutes=out_tzoffset))
-                            value = tz_localize_to_utc_single(value, tz)
-                            out_local = 0
-                            out_tzoffset = 0
-                        else:
-                            # Add a marker for naive string, to track if we are
-                            # parsing mixed naive and aware strings
-                            out_tzoffset_vals.add("naive")
-                        iresult[i] = value
-                        check_dts_bounds(&dts)
-
-                else:
-                    if is_coerce:
-                        iresult[i] = NPY_NAT
+                        raise TypeError(
+                            f"invalid string coercion to datetime "
+                            f"for \"{val}\", at position {i}"
+                        )
+
+                    if tz is not None:
+                        seen_datetime_offset = True
+                        # dateutil timezone objects cannot be hashed, so
+                        # store the UTC offsets in seconds instead
+                        out_tzoffset_vals.add(tz.total_seconds())
                     else:
-                        raise TypeError(f"{type(val)} is not convertible to datetime")
-
-            except OutOfBoundsDatetime as ex:
-                ex.args = (f"{ex}, at position {i}",)
-                if is_coerce:
-                    iresult[i] = NPY_NAT
-                    continue
-                raise
+                        # Add a marker for naive string, to track if we are
+                        # parsing mixed naive and aware strings
+                        out_tzoffset_vals.add("naive")
+
+                    _ts = convert_datetime_to_tsobject(py_dt, None)
+                    iresult[i] = _ts.value
+                if not string_to_dts_failed:
+                    # No error reported by string_to_dts, pick back up
+                    # where we left off
+                    value = npy_datetimestruct_to_datetime(NPY_FR_ns, &dts)
+                    if out_local == 1:
+                        seen_datetime_offset = True
+                        # Store the out_tzoffset in seconds
+                        # since we store the total_seconds of
+                        # dateutil.tz.tzoffset objects
+                        out_tzoffset_vals.add(out_tzoffset * 60.)
+                        tz = timezone(timedelta(minutes=out_tzoffset))
+                        value = tz_localize_to_utc_single(value, tz)
+                        out_local = 0
+                        out_tzoffset = 0
+                    else:
+                        # Add a marker for naive string, to track if we are
+                        # parsing mixed naive and aware strings
+                        out_tzoffset_vals.add("naive")
+                    iresult[i] = value
+                    check_dts_bounds(&dts)
 
-    except OutOfBoundsDatetime:
-        if is_raise:
-            raise
+            else:
+                raise TypeError(f"{type(val)} is not convertible to datetime")
 
-        return ignore_errors_out_of_bounds_fallback(values), tz_out
+        except (OutOfBoundsDatetime, ValueError) as ex:
+            ex.args = (f"{ex}, at position {i}",)
+            if is_coerce:
+                iresult[i] = NPY_NAT
+                continue
+            elif is_raise:
+                raise
+            return values, None
 
-    except TypeError:
-        return _array_to_datetime_object(values, errors, dayfirst, yearfirst)
+        except TypeError:
+            if is_coerce:
+                iresult[i] = NPY_NAT
+                continue
+            return _array_to_datetime_object(values, errors, dayfirst, yearfirst)
 
     if seen_datetime_offset and not utc_convert:
         # GH#17697
@@ -660,46 +651,6 @@ cpdef array_to_datetime(
     return result, tz_out
 
 
-@cython.wraparound(False)
-@cython.boundscheck(False)
-cdef ndarray[object] ignore_errors_out_of_bounds_fallback(ndarray[object] values):
-    """
-    Fallback for array_to_datetime if an OutOfBoundsDatetime is raised
-    and errors == "ignore"
-
-    Parameters
-    ----------
-    values : ndarray[object]
-
-    Returns
-    -------
-    ndarray[object]
-    """
-    cdef:
-        Py_ssize_t i, n = len(values)
-        object val
-
-    oresult = cnp.PyArray_EMPTY(values.ndim, values.shape, cnp.NPY_OBJECT, 0)
-
-    for i in range(n):
-        val = values[i]
-
-        # set as nan except if its a NaT
-        if checknull_with_nat_and_na(val):
-            if isinstance(val, float):
-                oresult[i] = np.nan
-            else:
-                oresult[i] = NaT
-        elif is_datetime64_object(val):
-            if get_datetime64_value(val) == NPY_NAT:
-                oresult[i] = NaT
-            else:
-                oresult[i] = val.item()
-        else:
-            oresult[i] = val
-    return oresult
-
-
 @cython.wraparound(False)
 @cython.boundscheck(False)
 cdef _array_to_datetime_object(
diff --git a/pandas/tests/tools/test_to_datetime.py b/pandas/tests/tools/test_to_datetime.py
@@ -1072,12 +1072,10 @@ def test_to_datetime_array_of_dt64s(self, cache, unit):
             ),
         )
 
-        # With errors='ignore', out of bounds datetime64s
-        # are converted to their .item(), which depending on the version of
-        # numpy is either a python datetime.datetime or datetime.date
+        # With errors='ignore', the input is returned
         tm.assert_index_equal(
             to_datetime(dts_with_oob, errors="ignore", cache=cache),
-            Index([dt.item() for dt in dts_with_oob]),
+            Index(dts_with_oob),
         )
 
     def test_to_datetime_tz(self, cache):
@@ -1094,8 +1092,9 @@ def test_to_datetime_tz(self, cache):
         )
         tm.assert_index_equal(result, expected)
 
-    def test_to_datetime_tz_mixed_raises(self, cache):
-        # mixed tzs will raise
+    def test_to_datetime_tz_mixed(self, cache):
+        # mixed tzs will raise if errors='raise'
+        # https://github.com/pandas-dev/pandas/issues/50585
         arr = [
             Timestamp("2013-01-01 13:00:00", tz="US/Pacific"),
             Timestamp("2013-01-02 14:00:00", tz="US/Eastern"),
@@ -1107,6 +1106,21 @@ def test_to_datetime_tz_mixed_raises(self, cache):
         with pytest.raises(ValueError, match=msg):
             to_datetime(arr, cache=cache)
 
+        result = to_datetime(arr, cache=cache, errors="ignore")
+        expected = Index(
+            [
+                Timestamp("2013-01-01 13:00:00-08:00"),
+                Timestamp("2013-01-02 14:00:00-05:00"),
+            ],
+            dtype="object",
+        )
+        tm.assert_index_equal(result, expected)
+        result = to_datetime(arr, cache=cache, errors="coerce")
+        expected = DatetimeIndex(
+            ["2013-01-01 13:00:00-08:00", "NaT"], dtype="datetime64[ns, US/Pacific]"
+        )
+        tm.assert_index_equal(result, expected)
+
     def test_to_datetime_different_offsets(self, cache):
         # inspired by asv timeseries.ToDatetimeNONISO8601 benchmark
         # see GH-26097 for more
