one in all

Author: charlesdong1991

doc/source/whatsnew/v0.25.0.rst

@@ -42,6 +42,8 @@ Other API Changes
 Deprecations
 ~~~~~~~~~~~~
 
+- :func:`Series.str.replace`, when pat is single special regex character and regex is not defined, regex is by default False for now, but this might be deprecated in the future.
+
 - Deprecated the `M (months)` and `Y (year)` `units` parameter of :func: `pandas.to_timedelta`, :func: `pandas.Timedelta` and :func: `pandas.TimedeltaIndex` (:issue:`16344`)
 
 .. _whatsnew_0250.prior_deprecations:
@@ -117,10 +119,8 @@ Conversion
 
 Strings
 ^^^^^^^
+- Bug in :func:`Series.str.replace` not applying regex in patterns of length 1 (:issue:`24804`)
 
--
--
--
 
 
 Interval

pandas/core/reshape/melt.py

@@ -420,7 +420,8 @@ def melt_stub(df, stub, i, j, value_vars, sep):
         newdf = melt(df, id_vars=i, value_vars=value_vars,
                      value_name=stub.rstrip(sep), var_name=j)
         newdf[j] = Categorical(newdf[j])
-        newdf[j] = newdf[j].str.replace(re.escape(stub + sep), "")
+        newdf[j] = newdf[j].str.replace(re.escape(stub + sep), "",
+                                        regex=True)
 
         # GH17627 Cast numerics suffixes to int/float
         newdf[j] = to_numeric(newdf[j], errors='ignore')

pandas/core/strings.py

@@ -425,7 +425,7 @@ def str_endswith(arr, pat, na=np.nan):
     return _na_map(f, arr, na, dtype=bool)
 
 
-def str_replace(arr, pat, repl, n=-1, case=None, flags=0, regex=True):
+def str_replace(arr, pat, repl, n=-1, case=None, flags=0, regex=None):
     r"""
     Replace occurrences of pattern/regex in the Series/Index with
     some other string. Equivalent to :meth:`str.replace` or
@@ -456,9 +456,13 @@ def str_replace(arr, pat, repl, n=-1, case=None, flags=0, regex=True):
     flags : int, default 0 (no flags)
         - re module flags, e.g. re.IGNORECASE
         - Cannot be set if `pat` is a compiled regex
-    regex : boolean, default True
+    regex : boolean, default None
         - If True, assumes the passed-in pattern is a regular expression.
         - If False, treats the pattern as a literal string
+        - If `pat` is a single character and `regex` is not specified, `pat`
+          is interpreted as a string literal. If `pat` is also a regular
+          expression symbol, a warning is issued that in the future `pat`
+          will be interpreted as a regex, rather than a literal.
         - Cannot be set to False if `pat` is a compiled regex or `repl` is
           a callable.
 
@@ -564,7 +568,7 @@ def str_replace(arr, pat, repl, n=-1, case=None, flags=0, regex=True):
             # add case flag, if provided
             if case is False:
                 flags |= re.IGNORECASE
-        if is_compiled_re or len(pat) > 1 or flags or callable(repl):
+        if is_compiled_re or pat or flags or callable(repl):
             n = n if n >= 0 else 0
             compiled = re.compile(pat, flags=flags)
             f = lambda x: compiled.sub(repl=repl, string=x, count=n)
@@ -577,6 +581,12 @@ def str_replace(arr, pat, repl, n=-1, case=None, flags=0, regex=True):
         if callable(repl):
             raise ValueError("Cannot use a callable replacement when "
                              "regex=False")
+        # if regex is default None, and a single special character is given
+        # in pat, still take it as a literal, and raise the Future warning
+        if regex is None and len(pat) == 1 and pat in list(r"[\^$.|?*+()]"):
+            warnings.warn("'{}' is interpreted as a literal in ".format(pat) +
+                          "default, not regex. It will change in the future.",
+                          FutureWarning)
         f = lambda x: x.replace(pat, repl, n)
 
     return _na_map(f, arr)
@@ -2529,7 +2539,7 @@ def match(self, pat, case=True, flags=0, na=np.nan):
         return self._wrap_result(result, fill_value=na)
 
     @copy(str_replace)
-    def replace(self, pat, repl, n=-1, case=None, flags=0, regex=True):
+    def replace(self, pat, repl, n=-1, case=None, flags=0, regex=None):
         result = str_replace(self._parent, pat, repl, n=n, case=case,
                              flags=flags, regex=regex)
         return self._wrap_result(result)

pandas/tests/test_strings.py

@@ -967,38 +967,39 @@ def test_casemethods(self):
     def test_replace(self):
         values = Series(['fooBAD__barBAD', NA])
 
-        result = values.str.replace('BAD[_]*', '')
+        result = values.str.replace('BAD[_]*', '', regex=True)
         exp = Series(['foobar', NA])
         tm.assert_series_equal(result, exp)
 
-        result = values.str.replace('BAD[_]*', '', n=1)
+        result = values.str.replace('BAD[_]*', '', regex=True, n=1)
         exp = Series(['foobarBAD', NA])
         tm.assert_series_equal(result, exp)
 
         # mixed
         mixed = Series(['aBAD', NA, 'bBAD', True, datetime.today(), 'fooBAD',
                         None, 1, 2.])
 
-        rs = Series(mixed).str.replace('BAD[_]*', '')
+        rs = Series(mixed).str.replace('BAD[_]*', '', regex=True)
         xp = Series(['a', NA, 'b', NA, NA, 'foo', NA, NA, NA])
         assert isinstance(rs, Series)
         tm.assert_almost_equal(rs, xp)
 
         # unicode
         values = Series([u('fooBAD__barBAD'), NA])
 
-        result = values.str.replace('BAD[_]*', '')
+        result = values.str.replace('BAD[_]*', '', regex=True)
         exp = Series([u('foobar'), NA])
         tm.assert_series_equal(result, exp)
 
-        result = values.str.replace('BAD[_]*', '', n=1)
+        result = values.str.replace('BAD[_]*', '', n=1, regex=True)
         exp = Series([u('foobarBAD'), NA])
         tm.assert_series_equal(result, exp)
 
         # flags + unicode
         values = Series([b"abcd,\xc3\xa0".decode("utf-8")])
         exp = Series([b"abcd, \xc3\xa0".decode("utf-8")])
-        result = values.str.replace(r"(?<=\w),(?=\w)", ", ", flags=re.UNICODE)
+        result = values.str.replace(r"(?<=\w),(?=\w)", ", ", regex=True,
+                                    flags=re.UNICODE)
         tm.assert_series_equal(result, exp)
 
         # GH 13438
@@ -1014,7 +1015,7 @@ def test_replace_callable(self):
 
         # test with callable
         repl = lambda m: m.group(0).swapcase()
-        result = values.str.replace('[a-z][A-Z]{2}', repl, n=2)
+        result = values.str.replace('[a-z][A-Z]{2}', repl, n=2, regex=True)
         exp = Series(['foObaD__baRbaD', NA])
         tm.assert_series_equal(result, exp)
 
@@ -1027,21 +1028,21 @@ def test_replace_callable(self):
 
         repl = lambda: None
         with pytest.raises(TypeError, match=p_err):
-            values.str.replace('a', repl)
+            values.str.replace('a', repl, regex=True)
 
         repl = lambda m, x: None
         with pytest.raises(TypeError, match=p_err):
-            values.str.replace('a', repl)
+            values.str.replace('a', repl, regex=True)
 
         repl = lambda m, x, y=None: None
         with pytest.raises(TypeError, match=p_err):
-            values.str.replace('a', repl)
+            values.str.replace('a', repl, regex=True)
 
         # test regex named groups
         values = Series(['Foo Bar Baz', NA])
         pat = r"(?P<first>\w+) (?P<middle>\w+) (?P<last>\w+)"
         repl = lambda m: m.group('middle').swapcase()
-        result = values.str.replace(pat, repl)
+        result = values.str.replace(pat, repl, regex=True)
         exp = Series(['bAR', NA])
         tm.assert_series_equal(result, exp)
 
@@ -1051,35 +1052,35 @@ def test_replace_compiled_regex(self):
 
         # test with compiled regex
         pat = re.compile(r'BAD[_]*')
-        result = values.str.replace(pat, '')
+        result = values.str.replace(pat, '', regex=True)
         exp = Series(['foobar', NA])
         tm.assert_series_equal(result, exp)
 
         # mixed
         mixed = Series(['aBAD', NA, 'bBAD', True, datetime.today(), 'fooBAD',
                         None, 1, 2.])
 
-        rs = Series(mixed).str.replace(pat, '')
+        rs = Series(mixed).str.replace(pat, '', regex=True)
         xp = Series(['a', NA, 'b', NA, NA, 'foo', NA, NA, NA])
         assert isinstance(rs, Series)
         tm.assert_almost_equal(rs, xp)
 
         # unicode
         values = Series([u('fooBAD__barBAD'), NA])
 
-        result = values.str.replace(pat, '')
+        result = values.str.replace(pat, '', regex=True)
         exp = Series([u('foobar'), NA])
         tm.assert_series_equal(result, exp)
 
-        result = values.str.replace(pat, '', n=1)
+        result = values.str.replace(pat, '', n=1, regex=True)
         exp = Series([u('foobarBAD'), NA])
         tm.assert_series_equal(result, exp)
 
         # flags + unicode
         values = Series([b"abcd,\xc3\xa0".decode("utf-8")])
         exp = Series([b"abcd, \xc3\xa0".decode("utf-8")])
         pat = re.compile(r"(?<=\w),(?=\w)", flags=re.UNICODE)
-        result = values.str.replace(pat, ", ")
+        result = values.str.replace(pat, ", ", regex=True)
         tm.assert_series_equal(result, exp)
 
         # case and flags provided to str.replace will have no effect
@@ -1089,29 +1090,30 @@ def test_replace_compiled_regex(self):
 
         with pytest.raises(ValueError,
                            match="case and flags cannot be"):
-            result = values.str.replace(pat, '', flags=re.IGNORECASE)
+            result = values.str.replace(pat, '', flags=re.IGNORECASE,
+                                        regex=True)
 
         with pytest.raises(ValueError,
                            match="case and flags cannot be"):
-            result = values.str.replace(pat, '', case=False)
+            result = values.str.replace(pat, '', case=False, regex=True)
 
         with pytest.raises(ValueError,
                            match="case and flags cannot be"):
-            result = values.str.replace(pat, '', case=True)
+            result = values.str.replace(pat, '', case=True, regex=True)
 
         # test with callable
         values = Series(['fooBAD__barBAD', NA])
         repl = lambda m: m.group(0).swapcase()
         pat = re.compile('[a-z][A-Z]{2}')
-        result = values.str.replace(pat, repl, n=2)
+        result = values.str.replace(pat, repl, n=2, regex=True)
         exp = Series(['foObaD__baRbaD', NA])
         tm.assert_series_equal(result, exp)
 
     def test_replace_literal(self):
         # GH16808 literal replace (regex=False vs regex=True)
         values = Series(['f.o', 'foo', NA])
         exp = Series(['bao', 'bao', NA])
-        result = values.str.replace('f.', 'ba')
+        result = values.str.replace('f.', 'ba', regex=True)
         tm.assert_series_equal(result, exp)
 
         exp = Series(['bao', 'foo', NA])
@@ -2915,6 +2917,7 @@ def test_partition_deprecation(self):
             result = values.str.rpartition(pat='_')
             tm.assert_frame_equal(result, expected)
 
+    @pytest.mark.filterwarnings("ignore: '|' is interpreted as a literal")
     def test_pipe_failures(self):
         # #2119
         s = Series(['A|B|C'])
@@ -2924,7 +2927,7 @@ def test_pipe_failures(self):
 
         tm.assert_series_equal(result, exp)
 
-        result = s.str.replace('|', ' ')
+        result = s.str.replace('|', ' ', regex=None)
         exp = Series(['A B C'])
 
         tm.assert_series_equal(result, exp)
@@ -3236,17 +3239,17 @@ def test_replace_moar(self):
         s = Series(['A', 'B', 'C', 'Aaba', 'Baca', '', NA, 'CABA',
                     'dog', 'cat'])
 
-        result = s.str.replace('A', 'YYY')
+        result = s.str.replace('A', 'YYY', regex=True)
         expected = Series(['YYY', 'B', 'C', 'YYYaba', 'Baca', '', NA,
                            'CYYYBYYY', 'dog', 'cat'])
         assert_series_equal(result, expected)
 
-        result = s.str.replace('A', 'YYY', case=False)
+        result = s.str.replace('A', 'YYY', case=False, regex=True)
         expected = Series(['YYY', 'B', 'C', 'YYYYYYbYYY', 'BYYYcYYY', '', NA,
                            'CYYYBYYY', 'dog', 'cYYYt'])
         assert_series_equal(result, expected)
 
-        result = s.str.replace('^.a|dog', 'XX-XX ', case=False)
+        result = s.str.replace('^.a|dog', 'XX-XX ', case=False, regex=True)
         expected = Series(['A', 'B', 'C', 'XX-XX ba', 'XX-XX ca', '', NA,
                            'XX-XX BA', 'XX-XX ', 'XX-XX t'])
         assert_series_equal(result, expected)
@@ -3424,3 +3427,36 @@ def test_method_on_bytes(self):
             expected = Series(np.array(
                 ['ad', 'be', 'cf'], 'S2').astype(object))
             tm.assert_series_equal(result, expected)
+
+    @pytest.mark.filterwarnings("ignore: '.' is interpreted as a literal")
+    @pytest.mark.parametrize("regex, expected_array", [
+        (True, ['foofoofoo', 'foofoofoo']),
+        (False, ['abc', '123']),
+        (None, ['abc', '123'])
+    ])
+    def test_replace_single_pattern(self, regex, expected_array):
+        values = Series(['abc', '123'])
+        # GH: 24804
+        result = values.str.replace('.', 'foo', regex=regex)
+        expected = Series(expected_array)
+        tm.assert_series_equal(result, expected)
+
+    @pytest.mark.parametrize("input_array, single_char, replace_char, "
+                             "expect_array, warn",
+                             [("a.c", ".", "b", "abc", True),
+                              ("a@c", "@", "at", "aatc", False)]
+                             )
+    def test_replace_warning_single_character(self, input_array,
+                                              single_char, replace_char,
+                                              expect_array, warn):
+        # GH: 24804
+        values = Series([input_array])
+        if warn:
+            with tm.assert_produces_warning(FutureWarning,
+                                            check_stacklevel=False):
+                result = values.str.replace(single_char, replace_char)
+        else:
+            result = values.str.replace(single_char, replace_char)
+
+        expected = Series([expect_array])
+        tm.assert_series_equal(result, expected)