diff --git a/changes/61d4b40b996d788ca2e3f3634843713c.yaml b/changes/61d4b40b996d788ca2e3f3634843713c.yaml new file mode 100644 index 0000000000..ebef3f0ba5 --- /dev/null +++ b/changes/61d4b40b996d788ca2e3f3634843713c.yaml @@ -0,0 +1,6 @@ +--- +desc: Added ``:plus`` and ``:base`` properties to ``inet:email``. +desc:literal: false +prs: [] +type: model +... diff --git a/changes/ba9dea8afe794ece1b40fa50f946abfd.yaml b/changes/ba9dea8afe794ece1b40fa50f946abfd.yaml new file mode 100644 index 0000000000..9a10c64d19 --- /dev/null +++ b/changes/ba9dea8afe794ece1b40fa50f946abfd.yaml @@ -0,0 +1,6 @@ +--- +desc: Migrated email addresses with + user names to properly populate ``:plus`` and ``:base``. +desc:literal: false +prs: [] +type: migration +... diff --git a/synapse/lib/modelrev.py b/synapse/lib/modelrev.py index eeb6d6a7ac..8d35ebd9cb 100644 --- a/synapse/lib/modelrev.py +++ b/synapse/lib/modelrev.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) -maxvers = (0, 2, 33) +maxvers = (0, 2, 34) class ModelRev: @@ -52,6 +52,7 @@ def __init__(self, core): ((0, 2, 31), self.revModel_0_2_31), ((0, 2, 32), self.revModel_0_2_32), ((0, 2, 33), self.revModel_0_2_33), + ((0, 2, 34), self.revModel_0_2_34), ) async def _uniqSortArray(self, todoprops, layers): @@ -824,6 +825,10 @@ async def revModel_0_2_32(self, layers): async def revModel_0_2_33(self, layers): await self._propToForm(layers, 'transport:sea:vessel:name', 'entity:name') + async def revModel_0_2_34(self, layers): + await self._normFormSubs(layers, 'inet:email') + await self._propToForm(layers, 'inet:email:base', 'inet:email') + async def runStorm(self, text, opts=None): ''' Run storm code in a schedcoro and log the output messages. diff --git a/synapse/models/inet.py b/synapse/models/inet.py index a5747fc2de..876f95d4ce 100644 --- a/synapse/models/inet.py +++ b/synapse/models/inet.py @@ -288,6 +288,11 @@ def _normPyStr(self, valu): mesg = f'Email address expected in @ format, got "{valu}"' raise s_exc.BadTypeValu(valu=valu, name=self.name, mesg=mesg) from None + plus = None + if len(parts := user.split('+', 1)) == 2: + baseuser, plus = parts + plus = plus.strip().lower() + try: fqdnnorm, fqdninfo = self.modl.type('inet:fqdn').norm(fqdn) usernorm, userinfo = self.modl.type('inet:user').norm(user) @@ -295,12 +300,18 @@ def _normPyStr(self, valu): raise s_exc.BadTypeValu(valu=valu, name=self.name, mesg=str(e)) from None norm = f'{usernorm}@{fqdnnorm}' + info = { 'subs': { 'fqdn': fqdnnorm, 'user': usernorm, } } + + if plus is not None: + info['subs']['plus'] = plus + info['subs']['base'] = f'{baseuser}@{fqdnnorm}' + return norm, info class Fqdn(s_types.Type): @@ -2117,12 +2128,22 @@ def getModelDefs(self): )), ('inet:email', {}, ( + ('user', ('inet:user', {}), { 'ro': True, 'doc': 'The username of the email address.'}), + ('fqdn', ('inet:fqdn', {}), { 'ro': True, 'doc': 'The domain of the email address.'}), + + ('plus', ('str', {'lower': True, 'strip': True}), { + 'ro': True, + 'doc': 'The optional email address "tag".'}), + + ('base', ('inet:email', {}), { + 'ro': True, + 'doc': 'The base email address which is populated if the email address contains a user with a +.'}), )), ('inet:flow', {}, ( diff --git a/synapse/tests/test_lib_modelrev.py b/synapse/tests/test_lib_modelrev.py index d66e92831e..6e818ea62e 100644 --- a/synapse/tests/test_lib_modelrev.py +++ b/synapse/tests/test_lib_modelrev.py @@ -1772,3 +1772,12 @@ async def test_modelrev_0_2_33(self): nodes = await core.nodes('entity:name') self.len(1, nodes) self.eq('foo bar', nodes[0].repr()) + + async def test_modelrev_0_2_34(self): + async with self.getRegrCore('model-0.2.34') as core: + nodes = await core.nodes('inet:email=visi+synapse@vertex.link') + self.len(1, nodes) + self.eq(nodes[0].get('user'), 'visi+synapse') + self.eq(nodes[0].get('plus'), 'synapse') + self.eq(nodes[0].get('base'), 'visi@vertex.link') + self.len(1, await core.nodes('inet:email=visi+synapse@vertex.link :base -> inet:email +inet:email=visi@vertex.link')) diff --git a/synapse/tests/test_model_inet.py b/synapse/tests/test_model_inet.py index c76fff72fe..dc7eb6b4fc 100644 --- a/synapse/tests/test_model_inet.py +++ b/synapse/tests/test_model_inet.py @@ -417,6 +417,62 @@ async def test_email(self): self.eq(node.get('fqdn'), 'vertex.link') self.eq(node.get('user'), 'unittest') + nodes = await core.nodes('[ inet:email=visi+Synapse@vertex.link ]') + self.len(1, nodes) + self.eq(nodes[0].ndef[1], 'visi+synapse@vertex.link') + self.eq(nodes[0].get('user'), 'visi+synapse') + self.eq(nodes[0].get('plus'), 'synapse') + self.eq(nodes[0].get('base'), 'visi@vertex.link') + self.len(1, await core.nodes('inet:email=visi+synapse@vertex.link :base -> inet:email +inet:email=visi@vertex.link')) + + nodes = await core.nodes('[ inet:email=visi++Synapse@vertex.link ]') + self.len(1, nodes) + self.eq(nodes[0].ndef[1], 'visi++synapse@vertex.link') + self.eq(nodes[0].get('user'), 'visi++synapse') + self.eq(nodes[0].get('plus'), '+synapse') + self.eq(nodes[0].get('base'), 'visi@vertex.link') + self.len(1, await core.nodes('inet:email=visi++synapse@vertex.link :base -> inet:email +inet:email=visi@vertex.link')) + + nodes = await core.nodes('[ inet:email=visi+Synapse+foo@vertex.link ]') + self.len(1, nodes) + self.eq(nodes[0].ndef[1], 'visi+synapse+foo@vertex.link') + self.eq(nodes[0].get('user'), 'visi+synapse+foo') + self.eq(nodes[0].get('plus'), 'synapse+foo') + self.eq(nodes[0].get('base'), 'visi@vertex.link') + self.len(1, await core.nodes('inet:email=visi+synapse+foo@vertex.link :base -> inet:email +inet:email=visi@vertex.link')) + + nodes = await core.nodes('[ inet:email=visi+@vertex.link ]') + self.len(1, nodes) + self.eq(nodes[0].ndef[1], 'visi+@vertex.link') + self.eq(nodes[0].get('user'), 'visi+') + self.eq(nodes[0].get('plus'), '') + self.eq(nodes[0].get('base'), 'visi@vertex.link') + self.len(1, await core.nodes('inet:email=visi+@vertex.link :base -> inet:email +inet:email=visi@vertex.link')) + + nodes = await core.nodes('[ inet:email=+@vertex.link ]') + self.len(1, nodes) + self.eq(nodes[0].ndef[1], '+@vertex.link') + self.eq(nodes[0].get('user'), '+') + self.eq(nodes[0].get('plus'), '') + self.eq(nodes[0].get('base'), '@vertex.link') + self.len(1, await core.nodes('inet:email="+@vertex.link" :base -> inet:email +inet:email="@vertex.link"')) + + nodes = await core.nodes('[ inet:email=++@vertex.link ]') + self.len(1, nodes) + self.eq(nodes[0].ndef[1], '++@vertex.link') + self.eq(nodes[0].get('user'), '++') + self.eq(nodes[0].get('plus'), '+') + self.eq(nodes[0].get('base'), '@vertex.link') + self.len(1, await core.nodes('inet:email="++@vertex.link" :base -> inet:email +inet:email="@vertex.link"')) + + nodes = await core.nodes('[ inet:email=+++@vertex.link ]') + self.len(1, nodes) + self.eq(nodes[0].ndef[1], '+++@vertex.link') + self.eq(nodes[0].get('user'), '+++') + self.eq(nodes[0].get('plus'), '++') + self.eq(nodes[0].get('base'), '@vertex.link') + self.len(1, await core.nodes('inet:email="+++@vertex.link" :base -> inet:email +inet:email="@vertex.link"')) + async def test_flow(self): async with self.getTestCore() as core: valu = s_common.guid()