diff --git a/pandas/core/algorithms.py b/pandas/core/algorithms.py
index 7fc391d3ffb51..3c7344b51cf41 100644
--- a/pandas/core/algorithms.py
+++ b/pandas/core/algorithms.py
@@ -47,7 +47,6 @@
     is_bool_dtype,
     is_complex_dtype,
     is_dict_like,
-    is_dtype_equal,
     is_extension_array_dtype,
     is_float,
     is_float_dtype,
@@ -55,7 +54,6 @@
     is_integer_dtype,
     is_list_like,
     is_object_dtype,
-    is_signed_integer_dtype,
     needs_i8_conversion,
 )
 from pandas.core.dtypes.concat import concat_compat
@@ -508,12 +506,26 @@ def isin(comps: ListLike, values: ListLike) -> npt.NDArray[np.bool_]:
         orig_values = list(values)
         values = _ensure_arraylike(orig_values, func_name="isin-targets")
 
-        if (
-            len(values) > 0
-            and values.dtype.kind in "iufcb"
-            and not is_signed_integer_dtype(comps)
-            and not is_dtype_equal(values, comps)
-        ):
+        try:
+            src = comps.dtype  # type: ignore[union-attr]
+            tar = values.dtype
+            # check only valid dtypes related to implicit conversion to float64
+            # other data types derived from 64-bit integers such as U/Int64Dtype
+            # should also work
+            if (
+                src.kind in "iu"
+                and src.itemsize == 8  # type: ignore[union-attr]
+                and tar.kind in "iu"
+                and tar.itemsize == 8  # type: ignore[union-attr]
+            ):
+                is_implicit_conversion_to_float64 = src != tar
+            else:
+                is_implicit_conversion_to_float64 = False
+        except (TypeError, AttributeError, ImportError):
+            # invalid comparison
+            is_implicit_conversion_to_float64 = False
+
+        if is_implicit_conversion_to_float64:
             # GH#46485 Use object to avoid upcast to float64 later
             # TODO: Share with _find_common_type_compat
             values = construct_1d_object_array_from_listlike(orig_values)
diff --git a/pandas/tests/test_algos.py b/pandas/tests/test_algos.py
index 7fb421e27bb40..ec87441e3941a 100644
--- a/pandas/tests/test_algos.py
+++ b/pandas/tests/test_algos.py
@@ -1197,6 +1197,13 @@ def test_isin_unsigned_dtype(self):
         expected = Series(False)
         tm.assert_series_equal(result, expected)
 
+    def test_isin_unsigned_dtype_other_side(self):
+        # GH#46485
+        ser = Series([1378774140726870442], dtype=np.int64)
+        result = ser.isin([np.uint64(1378774140726870528)])
+        expected = Series(False)
+        tm.assert_series_equal(result, expected)
+
 
 class TestValueCounts:
     def test_value_counts(self):