migr pushpull offs from global to hotcount (SYN-9345)

synapse/tests/test_tools_migrate3x.py

@@ -29,8 +29,8 @@
 class MigrationTest(s_t_utils.SynTest):
 
     @contextlib.asynccontextmanager
-    async def _getTestMigrCore(self, conf):
-        with self.getRegrDir('cortexes', REGR_CORE) as src:
+    async def _getTestMigrCore(self, conf, regrname=REGR_CORE):
+        with self.getRegrDir('cortexes', regrname) as src:
             with self.getTestDir(copyfrom=conf.get('dest')) as dest:
                 tconf = copy.deepcopy(conf)
                 tconf['src'] = src
@@ -103,3 +103,28 @@ async def test_migr_basic(self):
                 self.eq(nodes[0].get('input'), 'notenglish')
                 self.eq(nodes[0].get('output'), 'english')
                 self.eq(nodes[0].get('output:lang'), 'en')
+
+    async def test_migr_layeroffs(self):
+        conf = {
+            'src': None,
+            'dest': None,
+        }
+
+        async with self._getTestMigrCore(conf, regrname='pushpull-v2') as (migr, dest):
+            await migr.migrate()
+            await migr.fini()
+
+            async with await s_cortex.Cortex.anit(dest, conf=None) as core:
+
+                # test view has our nodes from the source cortex-view
+                nodes = await core.nodes('ps:contact', opts={'view': '41552988daf582ac7d05813a834e9c26'})
+                self.len(3, nodes)
+
+                # test offset is returned from the layer's hotcount
+                q = '$layer=$lib.layer.get($layr2) return ($layer)'
+                opts = {'vars': {'layr2': 'dd924b9a39f26638411a719dfff6caca'}}
+                layrinfo = await core.callStorm(q, opts=opts)
+                pulls = layrinfo.get('pulls')
+                self.len(1, pulls)
+                pdef = list(pulls.values())[0]
+                self.eq(23, pdef.get('offs', 0))

synapse/tools/migrate3x.py

@@ -391,6 +391,7 @@ async def _initStors(self, migr=True, cell=True):
 
         self.cortexdata = self.cellslab.getSafeKeyVal('cortex')
         self.cellinfo = self.cellslab.getSafeKeyVal('cell:info')
+        self.layeroffs = await self.cellslab.getHotCount('layeroffs')
         self.viewdefs = self.cortexdata.getSubKeyVal('view:info:')
         self.layrdefs = self.cortexdata.getSubKeyVal('layer:info:')
 
@@ -584,6 +585,22 @@ async def _migrCell(self):
 
             apptdefs.set(apptiden, info)
 
+        stormvars = self.cortexdata.getSubKeyVal('storm:vars:')
+        for iden, layrinfo in self.layrdefs.items():
+            logger.info(f'Migrating layer offsets for {iden}')
+            pushs = layrinfo.get('pushs', None)
+            pulls = layrinfo.get('pulls', None)
+            if pushs:
+                for push in pushs:
+                    offs = stormvars.pop(f'push:{push}', None)
+                    if offs is not None:
+                        self.layeroffs.set(push, offs)
+            if pulls:
+                for pull in pulls:
+                    offs = stormvars.pop(f'push:{pull}', None)
+                    if offs is not None:
+                        self.layeroffs.set(pull, offs)
+
         logger.info(f'Completed cell migration, removed deprecated confdefs: {remconfs}')
         await self._migrlogAdd('cell', 'prog', 'none', s_common.now())