Support for anonymous wildcards in pattern syntax: $_.

Once we have support for the concept, we can then support wildcards in places where we aren't yet comfortable implementing full support for named wildcards. In particular, repeated fields are easier to implement if they are unconstrained and unbound.

PiperOrigin-RevId: 568282402

Author: ssbr

refex/python/matchers/syntax_matchers.py

@@ -125,17 +125,18 @@
 from refex.python.matchers import base_matchers
 
 
-def _remap_macro_variables(pattern):
+def _remap_macro_variables(pattern: str) -> tuple[str, dict[str, str], set[str]]:
   """Renames the variables from the source pattern to give valid Python.
 
   Args:
     pattern: A source pattern containing metavariables like "$foo".
 
   Returns:
-    (remapped_source, variables)
+    (remapped_source, variables, anonymous_variables)
     * remapped_source is the pattern, but with all dollar-prefixed variables
       replaced with unique non-dollar-prefixed versions.
     * variables is the mapping of the original name to the remapped name.
+    * anonymous_variables is a set of remapped names that came from `_`.
 
   Raises:
     SyntaxError: The pattern can't be parsed.
@@ -147,6 +148,7 @@ def _remap_macro_variables(pattern):
       if i not in metavar_indices
   }
   original_to_unique = {}
+  anonymous_unique = set()
 
   for metavar_index in metavar_indices:
     metavar_token = list(remapped_tokens[metavar_index])
@@ -168,12 +170,19 @@ def _remap_macro_variables(pattern):
         remapped_name = 'gensym%s_%s' % (suffix, variable)
         if remapped_name not in taken_tokens:
           taken_tokens.add(remapped_name)
-          original_to_unique[variable] = remapped_name
+          if variable == '_':
+            anonymous_unique.add(remapped_name)
+          else:
+            original_to_unique[variable] = remapped_name
           break
     metavar_token[1] = remapped_name
     remapped_tokens[metavar_index] = tuple(metavar_token)
 
-  return tokenize.untokenize(remapped_tokens), original_to_unique
+  return (
+      tokenize.untokenize(remapped_tokens),
+      original_to_unique,
+      anonymous_unique,
+  )
 
 
 def _rewrite_submatchers(pattern, restrictions):
@@ -190,20 +199,22 @@ def _rewrite_submatchers(pattern, restrictions):
       valid Python syntax.
     * variables is the mapping of the original name to the remapped name.
     * new_submatchers is a dict from remapped names to submatchers. Every
-      variable is put in a Bind() node, which has a submatcher taken from
-      `restrictions`.
+      non-anonymous variable is put in a Bind() node, which has a submatcher
+      taken from `restrictions`.
 
   Raises:
     KeyError: if restrictions has a key that isn't a variable name.
   """
-  pattern, variables = _remap_macro_variables(pattern)
+  pattern, variables, anonymous_remapped = _remap_macro_variables(pattern)
   incorrect_variables = set(restrictions) - set(variables)
   if incorrect_variables:
     raise KeyError('Some variables specified in restrictions were missing. '
                    'Did you misplace a "$"? Missing variables: %r' %
                    incorrect_variables)
 
-  submatchers = {}
+  submatchers = {
+      new_name: base_matchers.Anything() for new_name in anonymous_remapped
+  }
   for old_name, new_name in variables.items():
     submatchers[new_name] = base_matchers.Bind(
         old_name,

refex/python/matchers/test_syntax_matchers.py

@@ -15,10 +15,6 @@
 # python3 python2
 """Tests for refex.python.matchers.syntax_matchers."""
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
 import textwrap
 import unittest
 from unittest import mock
@@ -59,21 +55,21 @@ def test_error_spacing_line(self):
 
   def test_identity(self):
     self.assertEqual(
-        syntax_matchers._remap_macro_variables('a + b'), ('a + b', {}))
+        syntax_matchers._remap_macro_variables('a + b'), ('a + b', {}, set())
+    )
 
   def test_remap(self):
     self.assertEqual(
-        syntax_matchers._remap_macro_variables('a + $b'), ('a + gensym_b', {
-            'b': 'gensym_b'
-        }))
+        syntax_matchers._remap_macro_variables('a + $b'),
+        ('a + gensym_b', {'b': 'gensym_b'}, set()),
+    )
 
   def test_remap_twice(self):
     # But why would you _do_ this?
     self.assertEqual(
         syntax_matchers._remap_macro_variables('gensym_b + $b'),
-        ('gensym_b + gensym0_b', {
-            'b': 'gensym0_b'
-        }))
+        ('gensym_b + gensym0_b', {'b': 'gensym0_b'}, set()),
+    )
 
   def test_remap_doesnt_eat_tokens(self):
     """Expanding the size of a variable mustn't eat into neighboring tokens."""
@@ -83,15 +79,16 @@ def test_remap_doesnt_eat_tokens(self):
         # columns to regenerate where things should go:
         # 1) eating whitespace: 'gensym_ain b'
         # 2) leavint the $ empty and causing a pahton indent: ' gensym_a in b'
-        ('gensym_a in b', {
-            'a': 'gensym_a'
-        }))
+        ('gensym_a in b', {'a': 'gensym_a'}, set()),
+    )
 
   def test_remap_is_noninvasive(self):
     """Remapping is lexical and doesn't invade comments or strings."""
     for s in ('# $cash', '"$money"'):
       with self.subTest(s=s):
-        self.assertEqual(syntax_matchers._remap_macro_variables(s), (s, {}))
+        self.assertEqual(
+            syntax_matchers._remap_macro_variables(s), (s, {}, set())
+        )
 
 
 class ExprPatternTest(matcher_test_util.MatcherTestCase):
@@ -153,7 +150,21 @@ def test_nonvariable_name_fails(self):
     expr = parsed.tree.body[0].value
     self.assertIsNone(
         syntax_matchers.ExprPattern('name').match(
-            matcher.MatchContext(parsed), expr))
+            matcher.MatchContext(parsed), expr
+        )
+    )
+
+  def test_anonymous_wildcard(self):
+    parsed = matcher.parse_ast('3', '<string>')
+    expr = parsed.tree.body[0].value
+    m = syntax_matchers.ExprPattern('$_').match(
+        matcher.MatchContext(parsed), expr
+    )
+    self.assertIsNotNone(m)
+    self.assertEqual(
+        m.bindings,
+        {},
+    )
 
   def test_variable_name(self):
     parsed = matcher.parse_ast('3', '<string>')