Backport PR #39615: BUG: fix Categorical.astype for dtype=np.int32 argument' (#39676)

Co-authored-by: Andrew Wieteska <[email protected]>

Author: simonjayhawkins

doc/source/whatsnew/v1.2.2.rst

@@ -21,6 +21,7 @@ Fixed regressions
 - Fixed regression in :meth:`~DataFrame.to_pickle` failing to create bz2/xz compressed pickle files with ``protocol=5`` (:issue:`39002`)
 - Fixed regression in :func:`pandas.testing.assert_series_equal` and :func:`pandas.testing.assert_frame_equal` always raising ``AssertionError`` when comparing extension dtypes (:issue:`39410`)
 - Fixed regression in :meth:`~DataFrame.to_csv` opening ``codecs.StreamWriter`` in binary mode instead of in text mode and ignoring user-provided ``mode`` (:issue:`39247`)
+- Fixed regression in :meth:`Categorical.astype` casting to incorrect dtype when ``np.int32`` is passed to dtype argument (:issue:`39402`)
 - Fixed regression in :meth:`~DataFrame.to_excel` creating corrupt files when appending (``mode="a"``) to an existing file (:issue:`39576`)
 - Fixed regression in :meth:`DataFrame.transform` failing in case of an empty DataFrame or Series (:issue:`39636`)
 - Fixed regression in :meth:`core.window.rolling.Rolling.count` where the ``min_periods`` argument would be set to ``0`` after the operation (:issue:`39554`)

pandas/conftest.py

@@ -1188,6 +1188,32 @@ def any_nullable_int_dtype(request):
     return request.param
 
 
+@pytest.fixture(params=tm.ALL_INT_DTYPES + tm.ALL_EA_INT_DTYPES)
+def any_int_or_nullable_int_dtype(request):
+    """
+    Parameterized fixture for any nullable integer dtype.
+
+    * int
+    * 'int8'
+    * 'uint8'
+    * 'int16'
+    * 'uint16'
+    * 'int32'
+    * 'uint32'
+    * 'int64'
+    * 'uint64'
+    * 'UInt8'
+    * 'Int8'
+    * 'UInt16'
+    * 'Int16'
+    * 'UInt32'
+    * 'Int32'
+    * 'UInt64'
+    * 'Int64'
+    """
+    return request.param
+
+
 @pytest.fixture(params=tm.ALL_EA_INT_DTYPES + tm.FLOAT_EA_DTYPES)
 def any_numeric_dtype(request):
     """

pandas/core/arrays/categorical.py

@@ -427,16 +427,16 @@ def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike:
         else:
             # GH8628 (PERF): astype category codes instead of astyping array
             try:
-                astyped_cats = self.categories.astype(dtype=dtype, copy=copy)
+                new_cats = np.asarray(self.categories)
+                new_cats = new_cats.astype(dtype=dtype, copy=copy)
             except (
                 TypeError,  # downstream error msg for CategoricalIndex is misleading
                 ValueError,
             ):
                 msg = f"Cannot cast {self.categories.dtype} dtype to {dtype}"
                 raise ValueError(msg)
 
-            astyped_cats = extract_array(astyped_cats, extract_numpy=True)
-            result = take_1d(astyped_cats, libalgos.ensure_platform_int(self._codes))
+            result = take_1d(new_cats, libalgos.ensure_platform_int(self._codes))
 
         return result
 

pandas/tests/arrays/categorical/test_dtypes.py

@@ -138,7 +138,7 @@ def test_astype(self, ordered):
         tm.assert_numpy_array_equal(result, expected)
 
         result = cat.astype(int)
-        expected = np.array(cat, dtype="int64")
+        expected = np.array(cat, dtype="int")
         tm.assert_numpy_array_equal(result, expected)
 
         result = cat.astype(float)

pandas/tests/series/test_dtypes.py

@@ -68,7 +68,7 @@ def test_astype_categorical_to_other(self):
         exp = Series(["a", "b", "b", "a", "a", "c", "c", "c"])
         tm.assert_series_equal(cat.astype("str"), exp)
         s2 = Series(Categorical(["1", "2", "3", "4"]))
-        exp2 = Series([1, 2, 3, 4]).astype("int64")
+        exp2 = Series([1, 2, 3, 4]).astype("int")
         tm.assert_series_equal(s2.astype("int"), exp2)
 
         # object don't sort correctly, so just compare that we have the same
@@ -109,6 +109,17 @@ def test_astype_categorical_invalid_conversions(self):
         with pytest.raises(TypeError, match=msg):
             ser.astype("object").astype(Categorical)
 
+    def test_categorical_astype_to_int(self, any_int_or_nullable_int_dtype):
+        # GH 39402
+
+        df = DataFrame(data={"col1": pd.array([2.0, 1.0, 3.0])})
+        df.col1 = df.col1.astype("category")
+        df.col1 = df.col1.astype(any_int_or_nullable_int_dtype)
+        expected = DataFrame(
+            {"col1": pd.array([2, 1, 3], dtype=any_int_or_nullable_int_dtype)}
+        )
+        tm.assert_frame_equal(df, expected)
+
     def test_series_to_categorical(self):
         # see gh-16524: test conversion of Series to Categorical
         series = Series(["a", "b", "c"])