add DataFrame conversion function

Author: skjerns

meg_utils/misc.py

@@ -669,6 +669,45 @@ def _safe_send(msg, parse_mode):
             pass
 
 
+def convert_to_numeric(df, convert_dtypes=True, inplace=True):
+    """
+    Convert DataFrame columns to numeric dtypes where possible.
+
+    Tries ``pd.to_numeric`` on every non-numeric column. If all values in a
+    column convert successfully, the numeric column is kept; otherwise the
+    original column is retained unchanged. Columns that contain any
+    non-numeric value (including ``None`` / ``NaN``) will not be converted.
+
+    Parameters
+    ----------
+    df : pandas.DataFrame
+        Input DataFrame.
+    convert_dtypes : bool, default=True
+        If True, call ``df.convert_dtypes()`` at the end to further optimize
+        dtypes (e.g. nullable integer types, string dtype).
+    inplace : bool, default=True
+        If True, modify *df* in place. If False, operate on a copy.
+
+    Returns
+    -------
+    pandas.DataFrame
+        DataFrame with eligible columns converted to numeric types.
+    """
+    if not inplace:
+        df = df.copy()
+    for col in df.columns:
+        if pd.api.types.is_numeric_dtype(df[col]):
+            continue
+        try:
+            converted = pd.to_numeric(df[col], errors='raise')
+            df[col] = converted
+        except ValueError:
+            pass
+    if convert_dtypes:
+        df = df.convert_dtypes()
+    return df
+
+
 def _fmt_duration(seconds):
     """Return human-readable duration."""
     seconds = float(seconds)

tests/test_misc.py

@@ -3,14 +3,15 @@
 """
 Tests for meg_utils.misc — focusing on to_long_df / long_df_to_array.
 """
-import sys; sys.path.append('../..')
+import sys, os
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import unittest
 import numpy as np
 import pandas as pd
 import pytest
 
-from meg_utils.misc import to_long_df, long_df_to_array
+from meg_utils.misc import to_long_df, long_df_to_array, convert_to_numeric
 
 
 # ---------------------------------------------------------------------------
@@ -212,5 +213,219 @@ def test_empty_columns_list(self):
             long_df_to_array(df, columns=[], value_name='v')
 
 
+# ---------------------------------------------------------------------------
+# convert_to_numeric
+# ---------------------------------------------------------------------------
+
+class TestConvertToNumeric:
+
+    # --- integer-like strings ---
+
+    def test_int_strings_converted(self):
+        """Columns of integer strings become numeric."""
+        df = pd.DataFrame({'a': ['1', '2', '3']})
+        out = convert_to_numeric(df)
+        assert pd.api.types.is_numeric_dtype(out['a'])
+        assert list(out['a']) == [1, 2, 3]
+
+    def test_negative_int_strings(self):
+        df = pd.DataFrame({'a': ['-5', '0', '10']})
+        out = convert_to_numeric(df)
+        assert pd.api.types.is_numeric_dtype(out['a'])
+        assert list(out['a']) == [-5, 0, 10]
+
+    # --- float-like strings ---
+
+    def test_float_strings_converted(self):
+        df = pd.DataFrame({'a': ['1.5', '2.7', '3.0']})
+        out = convert_to_numeric(df)
+        assert pd.api.types.is_numeric_dtype(out['a'])
+        np.testing.assert_allclose(out['a'].values, [1.5, 2.7, 3.0])
+
+    def test_scientific_notation(self):
+        df = pd.DataFrame({'a': ['1e3', '2.5e-1', '3E2']})
+        out = convert_to_numeric(df)
+        assert pd.api.types.is_numeric_dtype(out['a'])
+        np.testing.assert_allclose(out['a'].values, [1000.0, 0.25, 300.0])
+
+    # --- already numeric columns stay numeric ---
+
+    def test_int_column_unchanged(self):
+        df = pd.DataFrame({'a': [1, 2, 3]})
+        out = convert_to_numeric(df)
+        assert pd.api.types.is_numeric_dtype(out['a'])
+        assert list(out['a']) == [1, 2, 3]
+
+    def test_float_column_unchanged(self):
+        df = pd.DataFrame({'a': [1.1, 2.2, 3.3]})
+        out = convert_to_numeric(df)
+        assert pd.api.types.is_numeric_dtype(out['a'])
+
+    # --- non-convertible strings stay as strings ---
+
+    def test_pure_text_not_converted(self):
+        df = pd.DataFrame({'a': ['hello', 'world', 'foo']})
+        out = convert_to_numeric(df)
+        assert not pd.api.types.is_numeric_dtype(out['a'])
+        assert list(out['a']) == ['hello', 'world', 'foo']
+
+    def test_mixed_text_and_numbers_not_converted(self):
+        """If any value would become NaN, the whole column stays unchanged."""
+        df = pd.DataFrame({'a': ['1', '2', 'three']})
+        out = convert_to_numeric(df)
+        assert not pd.api.types.is_numeric_dtype(out['a'])
+
+    def test_partial_numeric_not_converted(self):
+        """Even a single non-numeric value blocks conversion."""
+        df = pd.DataFrame({'a': ['1.0', '2.0', 'N/A']})
+        out = convert_to_numeric(df)
+        assert not pd.api.types.is_numeric_dtype(out['a'])
+
+    # --- NaN / None handling ---
+    # errors='raise' means any None/NaN in a column blocks conversion
+
+    def test_column_with_none_and_numeric_strings_not_converted(self):
+        """None among numeric strings blocks conversion (errors='raise')."""
+        df = pd.DataFrame({'a': ['1', None, '3']})
+        out = convert_to_numeric(df)
+        assert not pd.api.types.is_numeric_dtype(out['a'])
+
+    def test_all_nan_column_not_converted(self):
+        """A column of all None is not converted to numeric."""
+        df = pd.DataFrame({'a': [None, None, None]})
+        out = convert_to_numeric(df)
+        assert not pd.api.types.is_numeric_dtype(out['a'])
+
+    def test_existing_nan_with_non_numeric_stays(self):
+        """NaN + non-numeric strings: column should not be converted."""
+        df = pd.DataFrame({'a': ['hello', None, 'world']})
+        out = convert_to_numeric(df)
+        assert not pd.api.types.is_numeric_dtype(out['a'])
+
+    def test_np_nan_in_numeric_strings_not_converted(self):
+        """np.nan among numeric strings also blocks conversion."""
+        df = pd.DataFrame({'a': ['1', np.nan, '3']})
+        out = convert_to_numeric(df)
+        assert not pd.api.types.is_numeric_dtype(out['a'])
+
+    # --- multiple columns ---
+
+    def test_mixed_columns(self):
+        """Each column is handled independently."""
+        df = pd.DataFrame({
+            'nums': ['10', '20', '30'],
+            'text': ['a', 'b', 'c'],
+            'floats': ['1.1', '2.2', '3.3'],
+            'mixed': ['1', 'x', '3'],
+            'ints': [4, 5, 6],
+        })
+        out = convert_to_numeric(df)
+        assert pd.api.types.is_numeric_dtype(out['nums'])
+        assert not pd.api.types.is_numeric_dtype(out['text'])
+        assert pd.api.types.is_numeric_dtype(out['floats'])
+        assert not pd.api.types.is_numeric_dtype(out['mixed'])
+        assert pd.api.types.is_numeric_dtype(out['ints'])
+
+    # --- boolean-like strings ---
+
+    def test_boolean_strings_not_numeric(self):
+        """'True'/'False' strings should not become numeric (they are not numbers)."""
+        df = pd.DataFrame({'a': ['True', 'False', 'True']})
+        out = convert_to_numeric(df)
+        assert not pd.api.types.is_numeric_dtype(out['a'])
+
+    # --- inplace parameter ---
+
+    def test_inplace_true_modifies_original(self):
+        """With inplace=True (default), the input DataFrame is mutated in place."""
+        df = pd.DataFrame({'a': ['1', '2', '3']})
+        convert_to_numeric(df, inplace=True, convert_dtypes=False)
+        # the column was converted on the original df
+        assert pd.api.types.is_numeric_dtype(df['a'])
+
+    def test_inplace_false_preserves_original(self):
+        """With inplace=False, the input DataFrame is not mutated."""
+        df = pd.DataFrame({'a': ['1', '2', '3'], 'b': ['x', 'y', 'z']})
+        df_orig = df.copy()
+        out = convert_to_numeric(df, inplace=False)
+        pd.testing.assert_frame_equal(df, df_orig)
+        assert pd.api.types.is_numeric_dtype(out['a'])
+
+    # --- empty DataFrame ---
+
+    def test_empty_dataframe(self):
+        df = pd.DataFrame()
+        out = convert_to_numeric(df)
+        assert out.empty
+
+    def test_dataframe_no_rows(self):
+        df = pd.DataFrame({'a': pd.Series([], dtype='object')})
+        out = convert_to_numeric(df)
+        assert len(out) == 0
+
+    # --- convert_dtypes parameter ---
+
+    def test_convert_dtypes_true_uses_nullable_int(self):
+        """With convert_dtypes=True, integer columns use nullable Int64."""
+        df = pd.DataFrame({'a': ['1', '2', '3']})
+        out = convert_to_numeric(df, convert_dtypes=True)
+        assert pd.api.types.is_integer_dtype(out['a'])
+
+    def test_convert_dtypes_false_skips_conversion(self):
+        """With convert_dtypes=False, no convert_dtypes() call is made."""
+        df = pd.DataFrame({'a': ['1', '2', '3'], 'b': ['x', 'y', 'z']})
+        out = convert_to_numeric(df, convert_dtypes=False)
+        assert pd.api.types.is_numeric_dtype(out['a'])
+        # 'b' should remain object dtype, not StringDtype
+        assert out['b'].dtype == object
+
+    def test_convert_dtypes_true_string_dtype(self):
+        """With convert_dtypes=True, text columns get StringDtype."""
+        df = pd.DataFrame({'a': ['hello', 'world']})
+        out = convert_to_numeric(df, convert_dtypes=True)
+        assert pd.api.types.is_string_dtype(out['a'])
+
+    # --- edge cases ---
+
+    def test_whitespace_strings_not_converted(self):
+        """Strings with only whitespace should not become numeric."""
+        df = pd.DataFrame({'a': ['  ', '\t', '\n']})
+        out = convert_to_numeric(df)
+        assert not pd.api.types.is_numeric_dtype(out['a'])
+
+    def test_numeric_with_whitespace_converted(self):
+        """Numeric strings with leading/trailing whitespace can still convert."""
+        df = pd.DataFrame({'a': [' 1 ', ' 2', '3 ']})
+        out = convert_to_numeric(df)
+        assert pd.api.types.is_numeric_dtype(out['a'])
+        assert list(out['a']) == [1, 2, 3]
+
+    def test_inf_strings_converted(self):
+        """'inf' and '-inf' are valid numeric values."""
+        df = pd.DataFrame({'a': ['inf', '-inf', '0']})
+        out = convert_to_numeric(df)
+        assert pd.api.types.is_numeric_dtype(out['a'])
+        assert np.isinf(out['a'].values[:2]).all()
+
+    def test_single_column_single_row(self):
+        df = pd.DataFrame({'a': ['42']})
+        out = convert_to_numeric(df)
+        assert pd.api.types.is_numeric_dtype(out['a'])
+        assert out['a'].iloc[0] == 42
+
+    def test_categorical_column_not_converted(self):
+        """Categorical string columns should not be converted to numeric."""
+        df = pd.DataFrame({'a': pd.Categorical(['x', 'y', 'z'])})
+        out = convert_to_numeric(df)
+        assert not pd.api.types.is_numeric_dtype(out['a'])
+
+    def test_categorical_numeric_stays(self):
+        """Categorical columns with numeric categories are already numeric-like."""
+        df = pd.DataFrame({'a': pd.Categorical([1, 2, 3])})
+        out = convert_to_numeric(df)
+        # should not error; exact dtype depends on convert_dtypes behavior
+        assert len(out) == 3
+
+
 if __name__ == "__main__":
     unittest.main(verbosity=2)