diff --git a/synapse/tests/test_tools_migrate3x.py b/synapse/tests/test_tools_migrate3x.py index 4d3bc70728a..acdfaff9e78 100644 --- a/synapse/tests/test_tools_migrate3x.py +++ b/synapse/tests/test_tools_migrate3x.py @@ -29,8 +29,8 @@ class MigrationTest(s_t_utils.SynTest): @contextlib.asynccontextmanager - async def _getTestMigrCore(self, conf): - with self.getRegrDir('cortexes', REGR_CORE) as src: + async def _getTestMigrCore(self, conf, regrname=REGR_CORE): + with self.getRegrDir('cortexes', regrname) as src: with self.getTestDir(copyfrom=conf.get('dest')) as dest: tconf = copy.deepcopy(conf) tconf['src'] = src @@ -103,3 +103,28 @@ async def test_migr_basic(self): self.eq(nodes[0].get('input'), 'notenglish') self.eq(nodes[0].get('output'), 'english') self.eq(nodes[0].get('output:lang'), 'en') + + async def test_migr_layeroffs(self): + conf = { + 'src': None, + 'dest': None, + } + + async with self._getTestMigrCore(conf, regrname='pushpull-v2') as (migr, dest): + await migr.migrate() + await migr.fini() + + async with await s_cortex.Cortex.anit(dest, conf=None) as core: + + # test view has our nodes from the source cortex-view + nodes = await core.nodes('ps:contact', opts={'view': '41552988daf582ac7d05813a834e9c26'}) + self.len(3, nodes) + + # test offset is returned from the layer's hotcount + q = '$layer=$lib.layer.get($layr2) return ($layer)' + opts = {'vars': {'layr2': 'dd924b9a39f26638411a719dfff6caca'}} + layrinfo = await core.callStorm(q, opts=opts) + pulls = layrinfo.get('pulls') + self.len(1, pulls) + pdef = list(pulls.values())[0] + self.eq(23, pdef.get('offs', 0)) diff --git a/synapse/tools/migrate3x.py b/synapse/tools/migrate3x.py index 8f381674e4e..b5bfd543d6a 100644 --- a/synapse/tools/migrate3x.py +++ b/synapse/tools/migrate3x.py @@ -391,6 +391,7 @@ async def _initStors(self, migr=True, cell=True): self.cortexdata = self.cellslab.getSafeKeyVal('cortex') self.cellinfo = self.cellslab.getSafeKeyVal('cell:info') + self.layeroffs = await self.cellslab.getHotCount('layeroffs') self.viewdefs = self.cortexdata.getSubKeyVal('view:info:') self.layrdefs = self.cortexdata.getSubKeyVal('layer:info:') @@ -584,6 +585,22 @@ async def _migrCell(self): apptdefs.set(apptiden, info) + stormvars = self.cortexdata.getSubKeyVal('storm:vars:') + for iden, layrinfo in self.layrdefs.items(): + logger.info(f'Migrating layer offsets for {iden}') + pushs = layrinfo.get('pushs', None) + pulls = layrinfo.get('pulls', None) + if pushs: + for push in pushs: + offs = stormvars.pop(f'push:{push}', None) + if offs is not None: + self.layeroffs.set(push, offs) + if pulls: + for pull in pulls: + offs = stormvars.pop(f'push:{pull}', None) + if offs is not None: + self.layeroffs.set(pull, offs) + logger.info(f'Completed cell migration, removed deprecated confdefs: {remconfs}') await self._migrlogAdd('cell', 'prog', 'none', s_common.now())