add wrappers.SoftmaxClickMouse

example/diagnostic-agent/diagnostic-agent.py

@@ -15,6 +15,9 @@
 
 logger = logging.getLogger()
 
+CHROME_X_OFFSET = 18
+CHROME_Y_OFFSET = 84
+
 class NoopSpace(gym.Space):
     """ Null action space """
     def sample(self, seed=0):
@@ -58,7 +61,7 @@ def __call__(self, observation, reward, done):
     universe.configure_logging()
 
     # Actions this agent will take, 'random' is the default
-    action_choices = ['random', 'noop', 'forward']
+    action_choices = ['random', 'noop', 'forward', 'click']
 
     parser = argparse.ArgumentParser(description=None)
     parser.add_argument('-e', '--env_id', default='gym-core.Pong-v3', help='Which environment to run on.')
@@ -103,6 +106,24 @@ def __call__(self, observation, reward, done):
     if args.monitor:
         env = wrappers.Monitor(env, '/tmp/vnc_random_agent', force=True)
 
+    if args.actions == 'random':
+        action_space = env.action_space
+    elif args.actions == 'noop':
+        action_space = NoopSpace()
+    elif args.actions == 'forward':
+        action_space = ForwardSpace()
+    elif args.actions == 'click':
+        spec = universe.runtime_spec('flashgames').server_registry[args.env_id]
+        height = spec["height"]
+        width = spec["width"]
+        noclick_regions = [r['coordinates'] for r in spec['regions'] if r['type'] == 'noclick'] if spec.get('regions') else []
+        active_region = (CHROME_X_OFFSET, CHROME_Y_OFFSET, CHROME_X_OFFSET + width, CHROME_Y_OFFSET + height)
+        env = wrappers.SoftmaxClickMouse(env, active_region=active_region, noclick_regions=noclick_regions)
+        action_space = env.action_space
+    else:
+        logger.error("Invalid action choice: {}".format(args.actions))
+        exit(1)
+
     env.configure(
         fps=args.fps,
         # print_frequency=None,
@@ -121,15 +142,6 @@ def __call__(self, observation, reward, done):
             'encoding': 'tight', 'compress_level': 0, 'fine_quality_level': 50, 'subsample_level': 0, 'quality_level': 5,
         },
     )
-    if args.actions == 'random':
-        action_space = env.action_space
-    elif args.actions == 'noop':
-        action_space = NoopSpace()
-    elif args.actions == 'forward':
-        action_space = ForwardSpace()
-    else:
-        logger.error("Invalid action choice: {}".format(args.actions))
-        exit(1)
 
     agent = RandomAgent(action_space, n=env.n, vectorized=env.metadata['runtime.vectorized'])
 

universe/wrappers/__init__.py

@@ -2,7 +2,7 @@
 import universe.wrappers.experimental
 from universe import envs, spaces
 from universe.wrappers import gym_core_sync
-from universe.wrappers.action_space import SafeActionSpace
+from universe.wrappers.action_space import SafeActionSpace, SoftmaxClickMouse
 from universe.wrappers.blocking_reset import BlockingReset
 from universe.wrappers.diagnostics import Diagnostics
 from universe.wrappers.gym_core import GymCoreAction, GymCoreObservation, CropAtari

universe/wrappers/action_space.py

@@ -52,6 +52,37 @@ def gym_core_action_space(gym_core_id):
         raise error.Error('Unsupported env type: {}'.format(spec.id))
 
 
+class SoftmaxClickMouse(vectorized.ActionWrapper):
+    def __init__(self, env, active_region=(10, 75 + 50, 10 + 160, 75 + 210), discrete_mouse_step=10, noclick_regions=[]):
+        super(SoftmaxClickMouse, self).__init__(env)
+        xlow, ylow, xhigh, yhigh = active_region
+        xs = range(xlow, xhigh, discrete_mouse_step)
+        ys = range(ylow, yhigh, discrete_mouse_step)
+        self._actions = []
+        removed = 0
+        for x in xs:
+            for y in ys:
+                xc = min(x+int(discrete_mouse_step/2), xhigh-1) # click to center of a cell
+                yc = min(y+int(discrete_mouse_step/2), yhigh-1)
+                if any(self.is_contained((xc, yc), r) for r in noclick_regions):
+                    removed += 1
+                    continue
+                e1 = spaces.PointerEvent(xc, yc, buttonmask=0) # release
+                e2 = spaces.PointerEvent(xc, yc, buttonmask=1) # click
+                e3 = spaces.PointerEvent(xc, yc, buttonmask=0) # release
+                self._actions.append([e1, e2, e3])
+        logger.info('noclick regions removed {} of {} actions'.format(removed, removed + len(self._actions)))
+        self.action_space = gym.spaces.Discrete(len(self._actions))
+
+    def _action(self, action_n):
+        return [self._actions[int(action)] for action in action_n]
+
+    @classmethod
+    def is_contained(cls, point, coords):
+        px, py = point
+        x, width, y, height = coords
+        return x <= px <= x + width and y <= py <= y + height
+
 
 class SafeActionSpace(vectorized.Wrapper):
     """