diff --git a/docs/synapse/userguides/storm_adv_functions.rstorm b/docs/synapse/userguides/storm_adv_functions.rstorm index cd99819d0a..94bc404299 100644 --- a/docs/synapse/userguides/storm_adv_functions.rstorm +++ b/docs/synapse/userguides/storm_adv_functions.rstorm @@ -439,7 +439,7 @@ simplifies this process. .. TIP:: - Synapse includes `gen.*`_ (generator) Storm commands and `$lib.gen`_ APIs that can generate many + Synapse includes `gen.*`_ (generator) Storm commands that can generate many common guid-based forms using secondary property deconfliction. @@ -775,6 +775,5 @@ keyword to yield any nodes into the pipeline; use a for loop to iterate over fun .. _guid-based: https://synapse.docs.vertex.link/en/latest/synapse/userguides/storm_ref_type_specific.html#guid .. _secondary property deconfliction: https://synapse.docs.vertex.link/en/latest/synapse/userguides/storm_ref_type_specific.html#guid-best-practices .. _gen.*: https://synapse.docs.vertex.link/en/latest/synapse/userguides/storm_ref_cmd.html#gen -.. _$lib.gen: https://synapse.docs.vertex.link/en/latest/synapse/autodocs/stormtypes_libs.html#lib-gen .. _privileged modules: https://synapse.docs.vertex.link/en/latest/synapse/devguides/power-ups.html#privileged-modules .. _Storm debugging tips: https://synapse.docs.vertex.link/en/latest/synapse/userguides/storm_adv_control.html#storm-debugging-tips diff --git a/docs/synapse/userguides/storm_ref_cmd.rstorm b/docs/synapse/userguides/storm_ref_cmd.rstorm index 95022e9288..02d78af964 100644 --- a/docs/synapse/userguides/storm_ref_cmd.rstorm +++ b/docs/synapse/userguides/storm_ref_cmd.rstorm @@ -1009,18 +1009,13 @@ an organization node deconflicted and created based on a name will only have its property set). Users can set additional property values as they see fit. - `gen.geo.place`_ -- `gen.it.av.scan.result`_ - `gen.it.software`_ - `gen.lang.language`_ - `gen.entity.campaign`_ -- `gen.ou.id.number`_ -- `gen.ou.id.type`_ - `gen.ou.industry`_ - `gen.ou.org`_ -- `gen.ou.org.hq`_ - `gen.pol.country`_ - `gen.pol.country.government`_ -- `gen.ps.contact.email`_ - `gen.risk.threat`_ - `gen.risk.tool.software`_ - `gen.risk.vuln`_ @@ -1049,22 +1044,6 @@ name (``geo:place:name`` and / or ``geo:place:names`` properties). .. storm-cli:: gen.geo.place --help -.. _storm-gen-it-av-scan-result: - -gen.it.av.scan.result -+++++++++++++++++++++ - -The ``gen.it.av.scan.result`` command locates (lifts) or creates an ``it:av:scan:result`` node based -on the form and value of the object scanned and the signature name used for the scan. - -You can optionally include the name of the scanner / scan engine and/or the time the scan was performed -for additional deconfliction. - -**Syntax:** - -.. storm-cli:: gen.it.av.scan.result --help - - .. _storm-gen-it-software: gen.it.software @@ -1105,32 +1084,6 @@ organization (``entity:campaign:reporter:name``). .. storm-cli:: gen.entity.campaign --help -.. _storm-gen-ou-id-number: - -gen.ou.id.number -++++++++++++++++ - -The ``gen.ou.id.number`` command locates (lifts) or creates an ``ou:id:number`` node based on -the organization ID type (``ou:id:type``) and organization ID value (``ou:id:value``). - -**Syntax:** - -.. storm-cli:: gen.ou.id.number --help - - -.. _storm-gen-ou-id-type: - -gen.ou.id.type -++++++++++++++ - -The ``gen.ou.id.type`` command locates (lifts) or creates an ``ou:id:type`` node based on -the friendly name of the organization ID type (``ou:id:type:name``). - -**Syntax:** - -.. storm-cli:: gen.ou.id.type --help - - .. _storm-gen-ou-industry: gen.ou.industry @@ -1157,23 +1110,6 @@ name (``ou:org:name`` and / or ``ou:org:names``). .. storm-cli:: gen.ou.org --help -.. _storm-gen-ou-org-hq: - -gen.ou.org.hq -+++++++++++++ - -The ``gen.ou.org.hq`` command locates (lifts) the primary ``ps:contact`` node for an organization -(i.e., the contact set for the ``ou:org:hq`` property) or creates the contact node (and sets the -``ou:org:hq`` property) if it does not exist, based on the organization name (``ou:org:name`` and / or -``ou:org:names``). - -**Syntax:** - -.. - FIXME - Correct this for 3.0.0 model changes when it is stable - .. storm-cli:: gen.ou.org.hq --help - - .. _storm-gen-pol-country: gen.pol.country @@ -1202,19 +1138,6 @@ on the two-letter ISO-3166 country code (``pol:country:iso2``). .. storm-cli:: gen.pol.country.government --help -.. _storm-gen-ps-contact-email: - -gen.ps.contact.email -++++++++++++++++++++ - -The ``gen.ps.contact.email`` command locates (lifts) or creates a ``ps:contact`` node using -the contact's primary email address (``ps:contact:email``) and type (``ps:contact:type``). - -**Syntax:** - -.. storm-cli:: gen.ps.contact.email --help - - .. _storm-gen-risk-threat: gen.risk.threat @@ -1251,7 +1174,7 @@ gen.risk.vuln The ``gen.risk.vuln`` command locates (lifts) or creates a ``risk:tool:vuln`` node using the Common Vulnerabilities and Exposures (CVE) number associated with the vulnerability -(``risk:vuln:cve``) and the name of the entity reporting on the vulnerability (``risk:vuln:reporter:name``). +(``risk:vuln:id``) and the name of the entity reporting on the vulnerability (``risk:vuln:reporter:name``). **Syntax:** diff --git a/synapse/lib/stormlib/gen.py b/synapse/lib/stormlib/gen.py index 37b305d235..1e07031b5a 100644 --- a/synapse/lib/stormlib/gen.py +++ b/synapse/lib/stormlib/gen.py @@ -5,429 +5,135 @@ class LibGen(s_stormtypes.Lib): ''' A Storm Library for secondary property based deconfliction. ''' - _storm_locals = ( - {'name': 'orgByName', 'desc': 'Returns an ou:org by name, adding the node if it does not exist.', - 'type': {'type': 'function', '_funcname': '_storm_query', - 'args': ( - {'name': 'name', 'type': 'str', 'desc': 'The name of the org.'}, - ), - 'returns': {'type': 'node', 'desc': 'An ou:org node with the given name.'}}}, - {'name': 'orgByFqdn', 'desc': 'Returns an ou:org node by FQDN, adding the node if it does not exist.', - 'type': {'type': 'function', '_funcname': '_storm_query', - 'args': ( - {'name': 'fqdn', 'type': 'str', 'desc': 'The FQDN of the org.'}, - {'name': 'try', 'type': 'boolean', 'default': False, - 'desc': 'Type normalization will fail silently instead of raising an exception.'}, - ), - 'returns': {'type': 'node', 'desc': 'An ou:org node with the given FQDN.'}}}, - {'name': 'industryByName', 'desc': 'Returns an ou:industry by name, adding the node if it does not exist.', - 'type': {'type': 'function', '_funcname': '_storm_query', - 'args': ( - {'name': 'name', 'type': 'str', 'desc': 'The name of the industry.'}, - ), - 'returns': {'type': 'node', 'desc': 'An ou:industry node with the given name.'}}}, - {'name': 'newsByUrl', 'desc': 'Returns a media:news node by URL, adding the node if it does not exist.', - 'type': {'type': 'function', '_funcname': '_storm_query', - 'args': ( - {'name': 'url', 'type': 'inet:url', 'desc': 'The URL where the news is published.'}, - {'name': 'try', 'type': 'boolean', 'default': False, - 'desc': 'Type normalization will fail silently instead of raising an exception.'}, - ), - 'returns': {'type': 'node', 'desc': 'A media:news node with the given URL.'}}}, - {'name': 'softByName', 'desc': 'Returns it:software node by name, adding the node if it does not exist.', - 'type': {'type': 'function', '_funcname': '_storm_query', - 'args': ( - {'name': 'name', 'type': 'str', 'desc': 'The name of the software.'}, - ), - 'returns': {'type': 'node', 'desc': 'An it:software node with the given name.'}}}, - {'name': 'vulnByCve', 'desc': 'Returns risk:vuln node by CVE and reporter, adding the node if it does not exist.', - 'type': {'type': 'function', '_funcname': '_storm_query', - 'args': ( - {'name': 'cve', 'type': 'str', 'desc': 'The CVE id.'}, - {'name': 'try', 'type': 'boolean', 'default': False, - 'desc': 'Type normalization will fail silently instead of raising an exception.'}, - {'name': 'reporter', 'type': 'str', 'default': None, - 'desc': 'The name of the organization which reported the vulnerability.'}, - ), - 'returns': {'type': 'node', 'desc': 'A risk:vuln node with the given CVE.'}}}, - - {'name': 'riskThreat', - 'desc': 'Returns a risk:threat node based on the threat and reporter names, adding the node if it does not exist.', - 'type': {'type': 'function', '_funcname': '_storm_query', - 'args': ( - {'name': 'name', 'type': 'str', 'desc': 'The reported name of the threat cluster.'}, - {'name': 'reporter', 'type': 'str', 'desc': 'The name of the organization which reported the threat cluster.'}, - ), - 'returns': {'type': 'node', 'desc': 'A risk:threat node.'}}}, - - {'name': 'riskToolSoftware', - 'desc': 'Returns a risk:tool:software node based on the tool and reporter names, adding the node if it does not exist.', - 'type': {'type': 'function', '_funcname': '_storm_query', - 'args': ( - {'name': 'name', 'type': 'str', 'desc': 'The reported name of the tool.'}, - {'name': 'reporter', 'type': 'str', 'desc': 'The name of the organization which reported the tool.'}, - ), - 'returns': {'type': 'node', 'desc': 'A risk:tool:software node.'}}}, - - {'name': 'psContactByEmail', 'desc': 'Returns a entity:contact by deconflicting the type and email address.', - 'type': {'type': 'function', '_funcname': '_storm_query', - 'args': ( - {'name': 'type', 'type': 'str', 'desc': 'The entity:contact:type property.'}, - {'name': 'email', 'type': 'str', 'desc': 'The entity:contact:email property.'}, - {'name': 'try', 'type': 'boolean', 'default': False, - 'desc': 'Type normalization will fail silently instead of raising an exception.'}, - ), - 'returns': {'type': 'node', 'desc': 'A entity:contact node.'}}}, - - {'name': 'polCountryByCode', 'desc': 'Returns a pol:country node by deconflicting the :code property.', - 'type': {'type': 'function', '_funcname': '_storm_query', - 'args': ( - {'name': 'code', 'type': 'str', 'desc': 'The pol:country:code property.'}, - {'name': 'try', 'type': 'boolean', 'default': False, - 'desc': 'Type normalization will fail silently instead of raising an exception.'}, - ), - 'returns': {'type': 'node', 'desc': 'A pol:country node.'}}}, - - {'name': 'langByName', 'desc': 'Returns a lang:language node by name, adding the node if it does not exist.', - 'type': {'type': 'function', '_funcname': '_storm_query', - 'args': ( - {'name': 'name', 'type': 'str', 'desc': 'The name of the language.'}, - ), - 'returns': {'type': 'node', 'desc': 'A lang:language node with the given name.'}}}, - {'name': 'langByCode', 'desc': 'Returns a lang:language node by language code, adding the node if it does not exist.', - 'type': {'type': 'function', '_funcname': '_storm_query', - 'args': ( - {'name': 'name', 'type': 'str', 'desc': 'The language code for the language.'}, - {'name': 'try', 'type': 'boolean', 'default': False, - 'desc': 'Type normalization will fail silently instead of raising an exception.'}, - ), - 'returns': {'type': 'node', 'desc': 'A lang:language node with the given code.'}}}, - {'name': 'campaign', - 'desc': 'Returns an entity:campaign node based on the campaign and reporter names, adding the node if it does not exist.', - 'type': {'type': 'function', '_funcname': '_storm_query', - 'args': ( - {'name': 'name', 'type': 'str', 'desc': 'The reported name of the campaign.'}, - {'name': 'reporter', 'type': 'str', 'desc': 'The name of the organization which reported the campaign.'}, - ), - 'returns': {'type': 'node', 'desc': 'An entity:campaign node.'}}}, - {'name': 'itAvScanResultByTarget', - 'desc': 'Returns an it:av:scan:result node by deconflicting with a target and signature name, adding the node if it does not exist.', - 'type': {'type': 'function', '_funcname': '_storm_query', - 'args': ( - {'name': 'form', 'type': 'str', 'desc': 'The target form.'}, - {'name': 'value', 'type': 'str', 'desc': 'The target value.'}, - {'name': 'signame', 'type': 'str', 'desc': 'The signature name.'}, - {'name': 'scanner', 'type': 'str', 'default': None, - 'desc': 'An optional scanner software name to include in deconfliction.'}, - {'name': 'time', 'type': 'time', 'default': None, - 'desc': 'An optional time when the scan was run to include in the deconfliction.'}, - {'name': 'try', 'type': 'boolean', 'default': False, - 'desc': 'Type normalization will fail silently instead of raising an exception.'}, - ), - 'returns': {'type': 'node', 'desc': 'An it:av:scan:result node.'}}}, - {'name': 'geoPlaceByName', 'desc': 'Returns a geo:place node by name, adding the node if it does not exist.', - 'type': {'type': 'function', '_funcname': '_storm_query', - 'args': ( - {'name': 'name', 'type': 'str', 'desc': 'The name of the place.'}, - ), - 'returns': {'type': 'node', 'desc': 'A geo:place node with the given name.'}}}, - {'name': 'fileBytesBySha256', - 'desc': 'Returns a file:bytes node by SHA256, adding the node if it does not exist.', - 'type': {'type': 'function', '_funcname': '_storm_query', - 'args': ( - {'name': 'sha256', 'type': 'str', 'desc': 'The SHA256 fingerprint for the file:bytes node.'}, - {'name': 'try', 'type': 'boolean', 'default': False, - 'desc': 'Type normalization will fail silently instead of raising an exception.'}, - ), - 'returns': {'type': 'node', 'desc': 'A file:bytes node with the given SHA256.'}}}, - {'name': 'cryptoX509CertBySha256', - 'desc': 'Returns a crypto:x509:cert node by SHA256, adding the node if it does not exist.', - 'type': {'type': 'function', '_funcname': '_storm_query', - 'args': ( - {'name': 'sha256', 'type': 'str', 'desc': 'The SHA256 fingerprint for the certificate.'}, - {'name': 'try', 'type': 'boolean', 'default': False, - 'desc': 'Type normalization will fail silently instead of raising an exception.'}, - ), - 'returns': {'type': 'node', 'desc': 'A crypto:x509:cert node with the given SHA256.'}}}, - {'name': 'inetTlsServerCertByServerAndSha256', - 'desc': 'Returns an inet:tls:servercert node by server and SHA256, adding the node if it does not exist.', - 'type': {'type': 'function', '_funcname': '_storm_query', - 'args': ( - {'name': 'server', 'type': ['str', 'inet:server'], 'desc': 'The server associated with the x509 certificate.'}, - {'name': 'sha256', 'type': 'str', 'desc': 'The SHA256 fingerprint for the certificate.'}, - {'name': 'try', 'type': 'boolean', 'default': False, - 'desc': 'Type normalization will fail silently instead of raising an exception.'}, - ), - 'returns': {'type': 'node', 'desc': 'An inet:tls:servercert node with the given server and SHA256.'}}}, - ) _storm_lib_path = ('gen',) _storm_query = ''' - function __maybeCast(try, type, valu) { + function __tryGutor(form, ctor, try) { if $try { - return($lib.trycast($type, $valu)) + return({[ *$form?=$ctor ]}) } - return(($lib.true, $lib.cast($type, $valu))) + return({[ *$form=$ctor ]}) } - function orgByName(name, try=$lib.false) { - if $try { - [ ou:org?=({"name": $name}) ] - } else { - [ ou:org=({"name": $name}) ] - } - return($node) + function _entityCampaignByName(name, reporter, try=(false)) { + $ctor = ({ + "name": $name, + "reporter:name": $reporter, + "$props": { + "reporter": $_ouOrgByName($reporter, try=$try), + }, + }) + return($__tryGutor(entity:campaign, $ctor, $try)) } - function orgByFqdn(fqdn, try=$lib.false) { - ($ok, $fqdn) = $__maybeCast($try, inet:fqdn, $fqdn) - if (not $ok) { return() } - - inet:fqdn=$fqdn -> ou:org - return($node) - - [ ou:org=(gen, fqdn, $fqdn) :dns:mx+=$fqdn ] - return($node) + function _geoPlaceByName(name, try=(false)) { + $ctor = ({ + "name": $name, + }) + return($__tryGutor(geo:place, $ctor, $try)) } - function industryByName(name) { - [ ou:industry=({"name": $name}) ] - return($node) + function _itSoftwareByName(name, try=(false)) { + $ctor = ({ + "name": $name, + }) + return($__tryGutor(it:software, $ctor, $try)) } - function newsByUrl(url, try=$lib.false) { - ($ok, $url) = $__maybeCast($try, inet:url, $url) - if (not $ok) { return() } - - doc:report:url=$url - return($node) - - [ doc:report=(gen, url, $url) :url=$url ] - return($node) + function _langLanguageByName(name, try=(false)) { + $ctor = ({ + "name": $name, + }) + return($__tryGutor(lang:language, $ctor, $try)) } - // FIXME remove? - function softByName(name) { - [ it:software=({"name": $name}) ] - return($node) + function _ouIndustryByName(name, reporter, try=(false)) { + $ctor = ({ + "name": $name, + "reporter:name": $reporter, + "$props": { + "reporter": $_ouOrgByName($reporter, try=$try), + }, + }) + return($__tryGutor(ou:industry, $ctor, $try)) } - // FIXME remove? - function vulnByCve(cve, try=$lib.false, reporter=$lib.null) { - ($ok, $cve) = $__maybeCast($try, it:sec:cve, $cve) - if (not $ok) { return() } - - risk:vuln:id={[ it:sec:cve=$cve ]} - if $reporter { - +:reporter:name=$reporter - { -:reporter [ :reporter=$orgByName($reporter) ] } - } - return($node) - - $guid = (gen, cve, $cve) - if $reporter { - $reporter = $lib.cast(meta:name, $reporter) - $guid.append($reporter) - } - - [ risk:vuln=$guid :id=$cve ] - if $reporter { - [ :reporter:name=$reporter :reporter=$orgByName($reporter) ] - } - return($node) + function _ouOrgByName(name, try=(false)) { + $ctor = ({ + "name": $name, + }) + return($__tryGutor(ou:org, $ctor, $try)) } - function riskThreat(name, reporter) { - meta:name=$name -> risk:threat - +:reporter:name=$reporter - { -:reporter [ :reporter=$orgByName($reporter) ] } - return($node) - - $name = $lib.cast(meta:name, $name) - $reporter = $lib.cast(meta:name, $reporter) - - [ risk:threat=(gen, name, reporter, $name, $reporter) - :name=$name - :reporter = { yield $orgByName($reporter) } - :reporter:name = $reporter - ] - return($node) + function _polCountryByCode(code, try=(false)) { + $ctor = ({ + "code": $code, + }) + return($__tryGutor(pol:country, $ctor, $try)) } - function riskToolSoftware(name, reporter) { - - meta:name = $name - -> risk:tool:software - +:reporter:name = $reporter - { -:reporter [ :reporter=$orgByName($reporter) ] } - return($node) - - $name = $lib.cast(meta:name, $name) - $reporter = $lib.cast(meta:name, $reporter) - - [ risk:tool:software=(gen, $name, $reporter) - :name = $name - :reporter:name = $reporter - :reporter = { yield $orgByName($reporter) } - ] - - return($node) - } - - function psContactByEmail(type, email, try=$lib.false) { - ($ok, $email) = $__maybeCast($try, inet:email, $email) - if (not $ok) { return() } - - ($ok, $type) = $__maybeCast($try, entity:contact:type:taxonomy, $type) - if (not $ok) { return() } - - entity:contact:email = $email - +:type = $type - return($node) - - [ entity:contact=(gen, type, email, $type, $email) - :email = $email - :type = $type - ] - return($node) - } - - function polCountryByCode(code, try=$lib.false) { - if $try { - ($ok, $code) = $lib.trycast(iso:3166:alpha2, $code) - if (not $ok) { return() } - } - [ pol:country=({"code": $code}) ] - return($node) - } - - function polCountryOrgByCode(code, try=$lib.false) { - - yield $lib.gen.polCountryByCode($code, try=$try) - - { -:government [ :government = $lib.gen.orgByName(`{:code} government`) ] } + function _polCountryOrgByCode(code, try=(false)) { + yield $_polCountryByCode($code, try=$try) + [ :government*unset=$_ouOrgByName(`{:code} government`) ] :government -> ou:org return($node) } - function langByName(name) { - [ lang:language=({"name": $name}) ] - return($node) - } - - function langByCode(code, try=(false)) { - if $try { - [ lang:language?=({"code": $code}) ] - } else { - [ lang:language=({"code": $code}) ] - } - return($node) - } - - function campaign(name, reporter) { - $reporg = {[ ou:org=({"name": $reporter}) ]} - [ entity:campaign=({"name": $name, "reporter": ["ou:org", $reporg]}) ] - [ :reporter:name*unset=$reporter ] - return($node) + function _riskThreatByName(name, reporter, try=(false)) { + $ctor = ({ + "name": $name, + "reporter:name": $reporter, + "$props": { + "reporter": $_ouOrgByName($reporter, try=$try), + }, + }) + return($__tryGutor(risk:threat, $ctor, $try)) } - function itAvScanResultByTarget(form, value, signame, scanner=$lib.null, time=$lib.null, try=$lib.false) { - - ($ok, $target) = $__maybeCast($try, it:av:scan:result:target, ($form, $value)) - if (not $ok) { return() } - - ($ok, $signame) = $__maybeCast($try, it:av:signame, $signame) - if (not $ok) { return() } - - $dict = ({"target": $target, "signame": $signame}) - - if ($scanner != $lib.null) { - ($ok, $scanner) = $__maybeCast($try, meta:name, $scanner) - if (not $ok) { return() } - $dict."scanner:name" = $scanner - } - - if ($time != $lib.null) { - ($ok, $time) = $__maybeCast($try, time, $time) - if (not $ok) { return() } - $dict.time = $time - } - - [ it:av:scan:result=$dict ] - - return($node) + function _riskToolSoftByName(name, reporter, try=(false)) { + $ctor = ({ + "name": $name, + "reporter:name": $reporter, + "$props": { + "reporter": $_ouOrgByName($reporter, try=$try), + }, + }) + return($__tryGutor(risk:tool:software, $ctor, $try)) } - function geoPlaceByName(name) { - $geoname = $lib.cast(meta:name, $name) - - meta:name=$geoname -> geo:place - return($node) - - [ geo:place=(gen, name, $geoname) :name=$geoname ] - return($node) - } - - // FIXME remove? - function fileBytesBySha256(sha256, try=$lib.false) { + function _riskVulnByCve(cve, reporter, try=(false)) { if $try { - [ file:bytes?=({"sha256": $sha256}) ] + $cve = {[ it:sec:cve?=$cve ]} + if ($cve = null) { return() } } else { - [ file:bytes=({"sha256": $sha256}) ] + $cve = {[ it:sec:cve=$cve ]} } - return($node) - } - function cryptoX509CertBySha256(sha256, try=$lib.false) { + $ctor = ({ + "id": $cve, + "reporter:name": $reporter, + "$props": { + "reporter": $_ouOrgByName($reporter, try=$try), + } + }) - ($ok, $sha256) = $lib.trycast(crypto:hash:sha256, $sha256) - if (not $ok and $try) { return() } - - $file = {[ file:bytes=({"sha256": $sha256}) ]} - - [ crypto:x509:cert=({"sha256": $sha256, "file": $file}) ] - - return($node) - } - - function inetTlsServerCertByServerAndSha256(server, sha256, try=$lib.false) { - ($ok, $server) = $__maybeCast($try, inet:server, $server) - if (not $ok) { return() } - - $crypto = $cryptoX509CertBySha256($sha256, try=$try) - if (not $crypto) { return() } - - [ inet:tls:servercert=($server, $crypto) ] - return($node) + return($__tryGutor(risk:vuln, $ctor, $try)) } ''' + +_tryarg = ('--try', {'help': 'Type normalization will fail silently instead of raising an exception.', + 'action': 'store_true'}) stormcmds = ( - { - 'name': 'gen.ou.id.number', - 'descr': 'Lift (or create) an ou:id:number node based on the organization ID type and value.', - 'cmdargs': ( - ('type', {'help': 'The type of the organization ID.'}), - ('value', {'help': 'The value of the organization ID.'}), - ), - 'storm': 'yield $lib.gen.orgIdNumber($cmdopts.type, $cmdopts.value)', - }, - { - 'name': 'gen.ou.id.type', - 'descr': 'Lift (or create) an ou:id:type node based on the name of the type.', - 'cmdargs': ( - ('name', {'help': 'The friendly name of the organization ID type.'}), - ), - 'storm': 'yield $lib.gen.orgIdType($cmdopts.name)', - }, { 'name': 'gen.ou.org', 'descr': 'Lift (or create) an ou:org node based on the organization name.', 'cmdargs': ( ('name', {'help': 'The name of the organization.'}), + _tryarg, ), - 'storm': 'yield $lib.gen.orgByName($cmdopts.name)', + 'storm': 'yield $lib.gen._ouOrgByName($cmdopts.name, try=$cmdopts.try)', }, { 'name': 'gen.entity.campaign', @@ -435,16 +141,18 @@ class LibGen(s_stormtypes.Lib): 'cmdargs': ( ('name', {'help': 'The name of the campaign.'}), ('reporter', {'help': 'The name of the reporting organization.'}), + _tryarg, ), - 'storm': 'yield $lib.gen.campaign($cmdopts.name, $cmdopts.reporter)', + 'storm': 'yield $lib.gen._entityCampaignByName($cmdopts.name, $cmdopts.reporter, try=$cmdopts.try)', }, { 'name': 'gen.it.software', 'descr': 'Lift (or create) an it:software node based on the software name.', 'cmdargs': ( ('name', {'help': 'The name of the software.'}), + _tryarg, ), - 'storm': 'yield $lib.gen.softByName($cmdopts.name)', + 'storm': 'yield $lib.gen._itSoftwareByName($cmdopts.name, try=$cmdopts.try)' }, { 'name': 'gen.risk.threat', @@ -459,8 +167,9 @@ class LibGen(s_stormtypes.Lib): 'cmdargs': ( ('name', {'help': 'The name of the threat cluster. For example: APT1'}), ('reporter', {'help': 'The name of the reporting organization. For example: Mandiant'}), + _tryarg, ), - 'storm': 'yield $lib.gen.riskThreat($cmdopts.name, $cmdopts.reporter)', + 'storm': 'yield $lib.gen._riskThreatByName($cmdopts.name, $cmdopts.reporter, try=$cmdopts.try)', }, { 'name': 'gen.risk.tool.software', @@ -475,8 +184,9 @@ class LibGen(s_stormtypes.Lib): 'cmdargs': ( ('name', {'help': 'The tool name.'}), ('reporter', {'help': 'The name of the reporting organization. For example: "recorded future"'}), + _tryarg, ), - 'storm': 'yield $lib.gen.riskToolSoftware($cmdopts.name, $cmdopts.reporter)', + 'storm': 'yield $lib.gen._riskToolSoftByName($cmdopts.name, $cmdopts.reporter, try=$cmdopts.try)', }, { 'name': 'gen.risk.vuln', @@ -490,11 +200,10 @@ class LibGen(s_stormtypes.Lib): ''', 'cmdargs': ( ('cve', {'help': 'The CVE identifier.'}), - ('reporter', {'help': 'The name of the reporting organization.', 'nargs': '?'}), - ('--try', {'help': 'Type normalization will fail silently instead of raising an exception.', - 'action': 'store_true'}), + ('reporter', {'help': 'The name of the reporting organization.'}), + _tryarg, ), - 'storm': 'yield $lib.gen.vulnByCve($cmdopts.cve, try=$cmdopts.try, reporter=$cmdopts.reporter)', + 'storm': 'yield $lib.gen._riskVulnByCve($cmdopts.cve, $cmdopts.reporter, try=$cmdopts.try)', }, { 'name': 'gen.ou.industry', @@ -503,8 +212,10 @@ class LibGen(s_stormtypes.Lib): ''', 'cmdargs': ( ('name', {'help': 'The industry name.'}), + ('reporter', {'help': 'The name of the reporting organization.'}), + _tryarg, ), - 'storm': 'yield $lib.gen.industryByName($cmdopts.name)', + 'storm': 'yield $lib.gen._ouIndustryByName($cmdopts.name, $cmdopts.reporter, try=$cmdopts.try)', }, { 'name': 'gen.pol.country', @@ -518,10 +229,9 @@ class LibGen(s_stormtypes.Lib): ''', 'cmdargs': ( ('code', {'help': 'The 2 letter ISO-3166 country code.'}), - ('--try', {'help': 'Type normalization will fail silently instead of raising an exception.', - 'action': 'store_true'}), + _tryarg, ), - 'storm': 'yield $lib.gen.polCountryByCode($cmdopts.code, try=$cmdopts.try)', + 'storm': 'yield $lib.gen._polCountryByCode($cmdopts.code, try=$cmdopts.try)', }, { 'name': 'gen.pol.country.government', @@ -535,65 +245,18 @@ class LibGen(s_stormtypes.Lib): ''', 'cmdargs': ( ('code', {'help': 'The 2 letter ISO-3166 country code.'}), - ('--try', {'help': 'Type normalization will fail silently instead of raising an exception.', - 'action': 'store_true'}), - ), - 'storm': 'yield $lib.gen.polCountryOrgByCode($cmdopts.code, try=$cmdopts.try)', - }, - { - 'name': 'gen.ps.contact.email', - 'descr': ''' - Lift (or create) the entity:contact node by deconflicting the email and type. - - Examples: - - // Yield the entity:contact node for the type and email - gen.ps.contact.email vertex.employee visi@vertex.link - ''', - 'cmdargs': ( - ('type', {'help': 'The contact type.'}), - ('email', {'help': 'The contact email address.'}), - ('--try', {'help': 'Type normalization will fail silently instead of raising an exception.', - 'action': 'store_true'}), + _tryarg, ), - 'storm': 'yield $lib.gen.psContactByEmail($cmdopts.type, $cmdopts.email, try=$cmdopts.try)', + 'storm': 'yield $lib.gen._polCountryOrgByCode($cmdopts.code, try=$cmdopts.try)', }, { 'name': 'gen.lang.language', 'descr': 'Lift (or create) a lang:language node based on the name.', 'cmdargs': ( ('name', {'help': 'The name of the language.'}), + _tryarg, ), - 'storm': 'yield $lib.gen.langByName($cmdopts.name)', - }, - { - 'name': 'gen.it.av.scan.result', - 'descr': ''' - Lift (or create) the it:av:scan:result node by deconflicting the target and signature name. - - The scan time and scanner name may also optionally be provided for deconfliction. - - Examples: - - // Yield the it:av:scan:result node for an FQDN and signature name - gen.it.av.scan.result inet:fqdn vertex.link foosig - - // Also deconflict by scanner name and scan time - gen.it.av.scan.result inet:fqdn fqdn vertex.link foosig --scanner-name barscanner --time 2022-11-03 - ''', - 'cmdargs': ( - ('form', {'help': 'The target form.'}), - ('value', {'help': 'The target value.'}), - ('signame', {'help': 'The signature name.'}), - ('--scanner-name', {'help': 'An optional scanner software name to include in deconfliction.'}), - ('--time', {'help': 'An optional time when the scan was run to include in the deconfliction.'}), - ('--try', {'help': 'Type normalization will fail silently instead of raising an exception.', - 'action': 'store_true'}), - ), - 'storm': ''' - yield $lib.gen.itAvScanResultByTarget($cmdopts.form, $cmdopts.value, $cmdopts.signame, - scanner=$cmdopts.scanner_name, time=$cmdopts.time, try=$cmdopts.try) - ''', + 'storm': 'yield $lib.gen._langLanguageByName($cmdopts.name, try=$cmdopts.try)', }, { 'name': 'gen.geo.place', @@ -602,7 +265,8 @@ class LibGen(s_stormtypes.Lib): ''', 'cmdargs': ( ('name', {'help': 'The name of the place.'}), + _tryarg, ), - 'storm': 'yield $lib.gen.geoPlaceByName($cmdopts.name)', + 'storm': 'yield $lib.gen._geoPlaceByName($cmdopts.name, try=$cmdopts.try)', }, ) diff --git a/synapse/lib/stormtypes.py b/synapse/lib/stormtypes.py index a0b9bd2a86..bce516526d 100644 --- a/synapse/lib/stormtypes.py +++ b/synapse/lib/stormtypes.py @@ -2596,7 +2596,7 @@ class LibLift(Lib): {'name': 'byNodeData', 'desc': 'Lift nodes which have a given nodedata name set on them.', 'type': {'type': 'function', '_funcname': '_byNodeData', 'args': ( - {'name': 'name', 'desc': 'The name to of the nodedata key to lift by.', 'type': 'str', }, + {'name': 'name', 'desc': 'The name of the nodedata key to lift by.', 'type': 'str', }, ), 'returns': {'name': 'Yields', 'type': 'node', 'desc': 'Yields nodes to the pipeline. ' diff --git a/synapse/tests/test_lib_ast.py b/synapse/tests/test_lib_ast.py index 1b5a0a9f9c..97c694fec2 100644 --- a/synapse/tests/test_lib_ast.py +++ b/synapse/tests/test_lib_ast.py @@ -3435,19 +3435,19 @@ async def test_ast_highlight(self): self.eq('obj.put(foo, bar, baz)', text[off:end]) self.stormIsInErr('pipe.put()', msgs) - text = '$lib.gen.campaign(foo, bar, baz)' + text = '$lib.lift.byNodeData(foo, bar, baz)' msgs = await core.stormlist(text) errm = [m for m in msgs if m[0] == 'err'][0] off, end = errm[1][1]['highlight']['offsets'] - self.eq('lib.gen.campaign(foo, bar, baz)', text[off:end]) - self.stormIsInErr('$lib.gen.campaign()', msgs) + self.eq('lib.lift.byNodeData(foo, bar, baz)', text[off:end]) + self.stormIsInErr('$lib.lift.byNodeData()', msgs) - text = '$gen = $lib.gen.campaign $gen(foo, bar, baz)' + text = '$lft = $lib.lift.byNodeData $lft(foo, bar, baz)' msgs = await core.stormlist(text) errm = [m for m in msgs if m[0] == 'err'][0] off, end = errm[1][1]['highlight']['offsets'] - self.eq('gen(foo, bar, baz)', text[off:end]) - self.stormIsInErr('$lib.gen.campaign()', msgs) + self.eq('lft(foo, bar, baz)', text[off:end]) + self.stormIsInErr('$lib.lift.byNodeData()', msgs) async def highlighteq(exp, text): msgs = await core.stormlist(text) diff --git a/synapse/tests/test_lib_storm.py b/synapse/tests/test_lib_storm.py index d77067be62..0a767ebef0 100644 --- a/synapse/tests/test_lib_storm.py +++ b/synapse/tests/test_lib_storm.py @@ -4324,12 +4324,12 @@ async def test_storm_help_cmd(self): msgs = await core.stormlist('$str=hehe help $str.split') self.stormIsInPrint('Split the string into multiple parts based on a separator.', msgs) - msgs = await core.stormlist('help $lib.gen.orgByName') - self.stormIsInPrint('Returns an ou:org by name, adding the node if it does not exist.', msgs) + msgs = await core.stormlist('help $lib.lift.byNodeData') + self.stormIsInPrint('Lift nodes which have a given nodedata name set on them.', msgs) - msgs = await core.stormlist('help --verbose $lib.gen.orgByName') - self.stormIsInPrint('Returns an ou:org by name, adding the node if it does not exist.\n' - 'Args:\n name (str): The name of the org.', msgs) + msgs = await core.stormlist('help --verbose $lib.lift.byNodeData') + self.stormIsInPrint('Lift nodes which have a given nodedata name set on them.\n' + 'Args:\n name (str): The name of the nodedata key to lift by.', msgs) orig = s_stormtypes.registry.getLibDocs def forcedep(cls): diff --git a/synapse/tests/test_lib_stormlib_gen.py b/synapse/tests/test_lib_stormlib_gen.py index 56fafe7b45..98c03ce184 100644 --- a/synapse/tests/test_lib_stormlib_gen.py +++ b/synapse/tests/test_lib_stormlib_gen.py @@ -8,305 +8,247 @@ class StormLibGenTest(s_test.SynTest): async def test_stormlib_gen(self): async with self.getTestCore() as core: - nodes00 = await core.nodes('yield $lib.gen.orgByName(vertex)') - nodes01 = await core.nodes('gen.ou.org vertex') - self.eq('vertex', nodes00[0].get('name')) - self.eq(nodes00[0].ndef, nodes01[0].ndef) - vtxguid = nodes00[0].ndef[1] - # FIXME discuss gen.ou.org.hq as ou:site + view2 = await core.callStorm('return($lib.view.add(layers=($lib.layer.add().iden,)).iden)') + onview2 = {'view': view2} - nodes00 = await core.nodes('yield $lib.gen.orgByFqdn(vertex.link)') - nodes01 = await core.nodes('yield $lib.gen.orgByFqdn(vertex.link)') - self.eq('vertex.link', nodes00[0].get('dns:mx')[0]) - self.eq(nodes00[0].ndef, nodes01[0].ndef) + self.len(1, nodes := await core.nodes('[ ou:org=({"name": "vtx"}) :names=(vertex,) ]')) + vtxguid = nodes[0].ndef[1] - self.len(0, await core.nodes('yield $lib.gen.orgByFqdn("...", try=$lib.true)')) + # gen.entity.campaign - nodes00 = await core.nodes('yield $lib.gen.industryByName(intelsoftware)') - nodes01 = await core.nodes('gen.ou.industry intelsoftware') - self.eq('intelsoftware', nodes00[0].get('name')) - self.eq(nodes00[0].ndef, nodes01[0].ndef) + self.len(0, await core.nodes('gen.entity.campaign overlord (null) --try')) + self.len(0, await core.nodes('gen.entity.campaign (null) vertex --try')) + await self.asyncraises(s_exc.BadTypeValu, core.nodes('gen.entity.campaign (null) vertex')) + await self.asyncraises(s_exc.BadTypeValu, core.nodes('gen.entity.campaign overload (null)')) - nodes00 = await core.nodes('yield $lib.gen.newsByUrl(https://vertex.link)') - nodes01 = await core.nodes('yield $lib.gen.newsByUrl(https://vertex.link)') - self.eq('https://vertex.link', nodes00[0].get('url')) + self.len(1, nodes00 := await core.nodes('gen.entity.campaign "operation overlord" vertex | [ :names+=overlord ]')) + self.len(1, nodes01 := await core.nodes('gen.entity.campaign overlord vertex')) + self.eq('operation overlord', nodes00[0].get('name')) + self.eq(['overlord'], nodes00[0].get('names')) + self.eq('vertex', nodes00[0].get('reporter:name')) + self.nn(nodes00[0].get('reporter')) self.eq(nodes00[0].ndef, nodes01[0].ndef) + self.eq(nodes00[0].getProps(), nodes01[0].getProps()) - self.len(0, await core.nodes('yield $lib.gen.newsByUrl("...", try=$lib.true)')) + self.len(1, nodes := await core.nodes('entity:campaign:name="operation overlord" :reporter -> ou:org')) + self.eq(vtxguid, nodes[0].ndef[1]) + self.eq(['vertex'], nodes[0].get('names')) - nodes00 = await core.nodes('yield $lib.gen.softByName(synapse)') - nodes01 = await core.nodes('gen.it.software synapse') - self.eq('synapse', nodes00[0].get('name')) - self.eq(nodes00[0].ndef, nodes01[0].ndef) + self.len(1, nodes := await core.nodes('gen.entity.campaign overlord otherorg')) + self.ne(nodes00[0].ndef[1], nodes[0].ndef[1]) + self.eq('overlord', nodes[0].get('name')) + self.none(nodes[0].get('names')) + self.eq('otherorg', nodes[0].get('reporter:name')) + self.nn(nodes[0].get('reporter')) - nodes00 = await core.nodes('yield $lib.gen.riskThreat(apt1, mandiant)') - nodes01 = await core.nodes('gen.risk.threat apt1 mandiant') - self.eq('apt1', nodes00[0].get('name')) - self.eq('mandiant', nodes00[0].get('reporter:name')) - self.eq(nodes00[0].ndef, nodes01[0].ndef) + self.len(1, nodes := await core.nodes('gen.entity.campaign "operation overlord" vertex', opts=onview2)) + self.eq(nodes00[0].ndef, nodes[0].ndef) - nodes00 = await core.nodes('yield $lib.gen.riskToolSoftware(redcat, vertex)') - nodes01 = await core.nodes('gen.risk.tool.software redcat vertex') - self.eq('redcat', nodes00[0].get('name')) - self.eq('vertex', nodes00[0].get('reporter:name')) - self.nn(nodes00[0].get('reporter')) - self.eq(nodes00[0].ndef, nodes01[0].ndef) + # gen.geo.place - nodes00 = await core.nodes('yield $lib.gen.vulnByCve(CVE-2022-00001)') - nodes01 = await core.nodes('gen.risk.vuln CVE-2022-00001') - self.eq(nodes00[0].ndef, nodes01[0].ndef) + self.len(0, await core.nodes('gen.geo.place (null) --try')) + await self.asyncraises(s_exc.BadTypeValu, core.nodes('gen.geo.place (null)')) - nodes00 = await core.nodes('yield $lib.gen.vulnByCve(CVE-2022-00001)') - nodes01 = await core.nodes('gen.risk.vuln CVE-2022-00001') + self.len(1, nodes00 := await core.nodes('gen.geo.place zimbabwe | [ :names+=rhodesia ]')) + self.len(1, nodes01 := await core.nodes('gen.geo.place rhodesia')) + self.eq('zimbabwe', nodes00[0].get('name')) + self.eq(['rhodesia'], nodes00[0].get('names')) self.eq(nodes00[0].ndef, nodes01[0].ndef) + self.eq(nodes00[0].getProps(), nodes01[0].getProps()) - self.len(1, await core.nodes('risk:vuln:id=CVE-2022-00001 [ :reporter:name=foo ]')) - nodes02 = await core.nodes('gen.risk.vuln CVE-2022-00001') - self.eq(nodes00[0].ndef, nodes02[0].ndef) + self.len(1, nodes := await core.nodes('gen.geo.place zimbabwe', opts=onview2)) + self.eq(nodes00[0].ndef, nodes[0].ndef) - nodes03 = await core.nodes('gen.risk.vuln CVE-2022-00001 foo') - self.eq(nodes00[0].ndef, nodes03[0].ndef) - self.nn(nodes03[0].get('reporter')) + # gen.it.software - nodes04 = await core.nodes('gen.risk.vuln CVE-2022-00001 bar') - nodes05 = await core.nodes('yield $lib.gen.vulnByCve(CVE-2022-00001, reporter=bar)') - self.eq(nodes04[0].ndef, nodes05[0].ndef) - self.ne(nodes00[0].ndef, nodes05[0].ndef) - self.eq('bar', nodes05[0].get('reporter:name')) - self.nn(nodes05[0].get('reporter')) + self.len(0, await core.nodes('gen.it.software (null) --try')) + await self.asyncraises(s_exc.BadTypeValu, core.nodes('gen.it.software (null)')) - self.len(0, await core.nodes('gen.risk.vuln newp --try')) - - nodes00 = await core.nodes('yield $lib.gen.polCountryByCode(UA)') - nodes01 = await core.nodes('gen.pol.country ua') + self.len(1, nodes00 := await core.nodes('gen.it.software rar | [ :names+=rarrr ]')) + self.len(1, nodes01 := await core.nodes('gen.it.software rarrr')) + self.eq('rar', nodes00[0].get('name')) + self.eq(['rarrr'], nodes00[0].get('names')) self.eq(nodes00[0].ndef, nodes01[0].ndef) + self.eq(nodes00[0].getProps(), nodes01[0].getProps()) - self.len(0, await core.nodes('gen.pol.country newp --try')) + self.len(1, nodes := await core.nodes('gen.it.software rar', opts=onview2)) + self.eq(nodes00[0].ndef, nodes[0].ndef) - self.len(1, await core.nodes(''' - gen.pol.country.government ua | - +ou:org +:name="ua government" - -> pol:country +:code=ua - ''')) + # gen.lang.language - self.len(0, await core.nodes('gen.pol.country.government newp --try')) + self.len(0, await core.nodes('gen.lang.language (null) --try')) + await self.asyncraises(s_exc.BadTypeValu, core.nodes('gen.lang.language (null)')) - nodes00 = await core.nodes('gen.ps.contact.email vertex.employee visi@vertex.link') - nodes01 = await core.nodes('yield $lib.gen.psContactByEmail(vertex.employee, visi@vertex.link)') - self.eq('vertex.employee.', nodes00[0].get('type')) - self.eq('visi@vertex.link', nodes00[0].get('email')) + self.len(1, nodes00 := await core.nodes('gen.lang.language german | [ :names+=deutsch ]')) + self.len(1, nodes01 := await core.nodes('gen.lang.language deutsch')) + self.eq('german', nodes00[0].get('name')) + self.eq(['deutsch'], nodes00[0].get('names')) self.eq(nodes00[0].ndef, nodes01[0].ndef) + self.eq(nodes00[0].getProps(), nodes01[0].getProps()) - self.len(0, await core.nodes('gen.ps.contact.email vertex.employee newp --try')) + self.len(1, nodes := await core.nodes('gen.lang.language german', opts=onview2)) + self.eq(nodes00[0].ndef, nodes[0].ndef) - nodes00 = await core.nodes('gen.lang.language "English (US)" | [ :names+="Murican" ]') - nodes01 = await core.nodes('yield $lib.gen.langByName(Murican)') - self.eq(nodes00[0].ndef, nodes01[0].ndef) + # gen.ou.industry - nodes00 = await core.nodes('gen.entity.campaign "operation overlord" vertex | [ :names+="d-day" ]') - nodes01 = await core.nodes('gen.entity.campaign d-day vertex') - nodes02 = await core.nodes('gen.entity.campaign d-day otherorg') - self.eq(nodes00[0].ndef, nodes01[0].ndef) - self.ne(nodes01[0].ndef, nodes02[0].ndef) + self.len(0, await core.nodes('gen.ou.industry ngo (null) --try')) + self.len(0, await core.nodes('gen.ou.industry (null) vertex --try')) + await self.asyncraises(s_exc.BadTypeValu, core.nodes('gen.ou.industry (null) vertex')) + await self.asyncraises(s_exc.BadTypeValu, core.nodes('gen.ou.industry ngo (null)')) + + self.len(1, nodes00 := await core.nodes('gen.ou.industry ngo vertex | [ :names+=ngos ]')) + self.len(1, nodes01 := await core.nodes('gen.ou.industry ngos vertex')) + self.eq('ngo', nodes00[0].get('name')) + self.eq(['ngos'], nodes00[0].get('names')) + self.eq('vertex', nodes00[0].get('reporter:name')) self.nn(nodes00[0].get('reporter')) - self.nn(nodes01[0].get('reporter')) - self.nn(nodes02[0].get('reporter')) - - q = 'gen.it.av.scan.result inet:fqdn vertex.link foosig --scanner-name barscn --time 2022' - nodes00 = await core.nodes(q) - self.len(1, nodes00) - self.eq(('inet:fqdn', 'vertex.link'), nodes00[0].get('target')) - self.eq('foosig', nodes00[0].get('signame')) - self.eq('barscn', nodes00[0].get('scanner:name')) - self.eq('2022-01-01T00:00:00Z', nodes00[0].repr('time')) - nodes01 = await core.nodes(q) self.eq(nodes00[0].ndef, nodes01[0].ndef) + self.eq(nodes00[0].getProps(), nodes01[0].getProps()) + + self.len(1, nodes := await core.nodes('ou:industry:name=ngo :reporter -> ou:org')) + self.eq(vtxguid, nodes[0].ndef[1]) + + self.len(1, nodes := await core.nodes('gen.ou.industry ngo otherorg')) + self.ne(nodes00[0].ndef[1], nodes[0].ndef[1]) + self.eq('ngo', nodes[0].get('name')) + self.none(nodes[0].get('names')) + self.eq('otherorg', nodes[0].get('reporter:name')) + self.nn(nodes[0].get('reporter')) + + self.len(1, nodes := await core.nodes('gen.ou.industry ngo vertex', opts=onview2)) + self.eq(nodes00[0].ndef, nodes[0].ndef) + + # gen.ou.org + + self.len(0, await core.nodes('gen.ou.org (null) --try')) + await self.asyncraises(s_exc.BadTypeValu, core.nodes('gen.ou.org (null)')) + + self.len(1, nodes00 := await core.nodes('gen.ou.org intel | [ :names+=intelsoft ]')) + self.len(1, nodes01 := await core.nodes('gen.ou.org intelsoft')) + self.eq('intel', nodes00[0].get('name')) + self.eq(['intelsoft'], nodes00[0].get('names')) + self.eq(nodes00[0].ndef, nodes01[0].ndef) + self.eq(nodes00[0].getProps(), nodes01[0].getProps()) + + self.len(1, nodes := await core.nodes('gen.ou.org intel', opts=onview2)) + self.eq(nodes00[0].ndef, nodes[0].ndef) + + # gen.pol.country + + self.len(0, await core.nodes('gen.pol.country newp --try')) + await self.asyncraises(s_exc.BadTypeValu, core.nodes('gen.pol.country newp')) - nodes02 = await core.nodes('gen.it.av.scan.result inet:fqdn vertex.link foosig --scanner-name barscn') - self.eq(nodes00[0].ndef, nodes02[0].ndef) - self.eq('2022-01-01T00:00:00Z', nodes02[0].repr('time')) - - nodes03 = await core.nodes('gen.it.av.scan.result inet:fqdn vertex.link foosig --scanner-name bazscn') - self.ne(nodes00[0].ndef, nodes03[0].ndef) - - nodes04 = await core.nodes('gen.it.av.scan.result inet:fqdn vertex.link foosig --time 2022') - self.eq(nodes00[0].ndef, nodes04[0].ndef) - - nodes05 = await core.nodes('gen.it.av.scan.result inet:fqdn vertex.link foosig --time 2023') - self.ne(nodes00[0].ndef, nodes05[0].ndef) - - opts = { - 'vars': { - 'guid': '28c5902d115f29f1fcb818c0abeaa491', - 'ip': '1.2.3.4', - 'fqdn': 'vtk.lk', - } - } - - self.len(1, await core.nodes('gen.it.av.scan.result file:bytes $guid foosig', opts=opts)) - self.len(1, await core.nodes('gen.it.av.scan.result inet:fqdn $fqdn foosig', opts=opts)) - self.len(1, await core.nodes('gen.it.av.scan.result inet:ip $ip foosig', opts=opts)) - self.len(1, await core.nodes('gen.it.av.scan.result inet:url `http://{$fqdn}` foosig', opts=opts)) - self.len(1, await core.nodes('gen.it.av.scan.result it:exec:proc $guid foosig', opts=opts)) - self.len(1, await core.nodes('gen.it.av.scan.result it:host $guid foosig', opts=opts)) - - self.len(6, await core.nodes(''' - file:bytes=$guid - inet:fqdn=$fqdn - it:host=$guid - inet:ip=$ip - it:exec:proc=$guid - inet:url=`http://{$fqdn}` - +{ - ($form, $valu) = $node.ndef() - -> { gen.it.av.scan.result $form $valu foosig } - }=1 - -> it:av:scan:result - ''', opts=opts)) - - with self.raises(s_exc.NoSuchForm) as cm: - await core.nodes('gen.it.av.scan.result newp vertex.link foosig --try') - self.eq('No form named newp.', cm.exception.errinfo['mesg']) - - with self.raises(s_exc.BadTypeValu) as cm: - await core.nodes('gen.it.av.scan.result meta:name nah foosig') - self.isin('Ndef of form meta:name is not allowed', cm.exception.errinfo['mesg']) - - self.len(0, await core.nodes('gen.it.av.scan.result file:bytes newp foosig --try')) - - self.none(await core.callStorm('return($lib.gen.itAvScanResultByTarget(inet:fqdn, vertex.link, $lib.null, try=$lib.true))')) - self.none(await core.callStorm('return($lib.gen.itAvScanResultByTarget(inet:fqdn, "..", barsig, try=$lib.true))')) - self.none(await core.callStorm('return($lib.gen.itAvScanResultByTarget(inet:fqdn, vertex.link, barsig, scanner=$lib.set(), try=$lib.true))')) - self.none(await core.callStorm('return($lib.gen.itAvScanResultByTarget(inet:fqdn, vertex.link, barsig, time=newp, try=$lib.true))')) - - # Stable guid test - fork = await core.callStorm('return( $lib.view.get().fork().iden )') - - nodes00 = await core.nodes('yield $lib.gen.orgByName(forkOrg)', opts={'view': fork}) - self.len(1, nodes00) - nodes01 = await core.nodes('yield $lib.gen.orgByName(forkOrg)') - self.len(1, nodes01) + self.len(1, nodes00 := await core.nodes('gen.pol.country us')) + self.len(1, nodes01 := await core.nodes('gen.pol.country us')) + self.eq('us', nodes00[0].get('code')) self.eq(nodes00[0].ndef, nodes01[0].ndef) + self.eq(nodes00[0].getProps(), nodes01[0].getProps()) - nodes02 = await core.nodes('yield $lib.gen.orgByName(anotherForkOrg)', opts={'view': fork}) - self.len(1, nodes02) - self.len(0, await core.nodes('ou:org:name=anotherforkorg')) + self.len(1, nodes := await core.nodes('gen.pol.country us', opts=onview2)) + self.eq(nodes00[0].ndef, nodes[0].ndef) - # Merge the fork down - await core.nodes('view.merge --delete $fork', opts={'vars': {'fork': fork}}) + # gen.pol.country.government - self.len(1, await core.nodes('ou:org:name=forkorg')) - self.len(1, await core.nodes('ou:org:name=anotherforkorg')) + self.len(0, await core.nodes('gen.pol.country.government newp --try')) + await self.asyncraises(s_exc.BadTypeValu, core.nodes('gen.pol.country.government newp')) - nodes = await core.nodes('geo:place') - self.len(0, nodes) + self.len(1, nodes00 := await core.nodes('gen.pol.country.government us')) + self.len(1, nodes01 := await core.nodes('gen.pol.country.government us')) + self.eq('ou:org', nodes00[0].ndef[0]) + self.eq('us government', nodes00[0].get('name')) + self.eq(nodes00[0].ndef, nodes01[0].ndef) + self.eq(nodes00[0].getProps(), nodes01[0].getProps()) - nodes = await core.nodes('gen.geo.place Zimbabwe') - self.len(1, nodes) - self.eq(nodes[0].get('name'), 'zimbabwe') - self.none(nodes[0].get('names')) + self.len(1, pols00 := await core.nodes('ou:org:name="us government" -> pol:country')) + self.eq('us', pols00[0].get('code')) - iden = nodes[0].iden() + self.len(1, nodes := await core.nodes('gen.pol.country.government us', opts=onview2)) + self.eq(nodes00[0].ndef, nodes[0].ndef) + self.len(1, nodes := await core.nodes('ou:org:name="us government" -> pol:country', opts=onview2)) + self.eq(pols00[0].ndef, nodes[0].ndef) - msgs = await core.stormlist('geo:place:name=zimbabwe [ :names+=Rhodesia ]') - self.stormHasNoWarnErr(msgs) + # gen.risk.threat - nodes = await core.nodes('gen.geo.place Rhodesia') - self.len(1, nodes) - self.eq(nodes[0].iden(), iden) - names = nodes[0].get('names') - self.len(1, names) - self.isin('rhodesia', names) + self.len(0, await core.nodes('gen.risk.threat apt1 (null) --try')) + self.len(0, await core.nodes('gen.risk.threat (null) vertex --try')) + await self.asyncraises(s_exc.BadTypeValu, core.nodes('gen.risk.threat (null) vertex')) + await self.asyncraises(s_exc.BadTypeValu, core.nodes('gen.risk.threat apt1 (null)')) - async def test_stormlib_gen_fileBytes(self): + self.len(1, nodes00 := await core.nodes('gen.risk.threat apt1 vertex | [ :names+=apt-1 ]')) + self.len(1, nodes01 := await core.nodes('gen.risk.threat apt-1 vertex')) + self.eq('apt1', nodes00[0].get('name')) + self.eq(['apt-1'], nodes00[0].get('names')) + self.eq('vertex', nodes00[0].get('reporter:name')) + self.nn(nodes00[0].get('reporter')) + self.eq(nodes00[0].ndef, nodes01[0].ndef) + self.eq(nodes00[0].getProps(), nodes01[0].getProps()) - async with self.getTestCore() as core: - sha256 = s_common.buid().hex() - opts = {'vars': {'sha256': sha256}} + self.len(1, nodes := await core.nodes('risk:threat:name=apt1 :reporter -> ou:org')) + self.eq(vtxguid, nodes[0].ndef[1]) - nodes = await core.nodes('yield $lib.gen.fileBytesBySha256($sha256)', opts=opts) - self.len(1, nodes) - self.eq(nodes[0].get('sha256'), sha256) + self.len(1, nodes := await core.nodes('gen.risk.threat apt1 otherorg')) + self.ne(nodes00[0].ndef[1], nodes[0].ndef[1]) + self.eq('apt1', nodes[0].get('name')) + self.none(nodes[0].get('names')) + self.eq('otherorg', nodes[0].get('reporter:name')) + self.nn(nodes[0].get('reporter')) - sha256 = s_common.buid().hex() - opts = {'vars': {'sha256': sha256}} + self.len(1, nodes := await core.nodes('gen.risk.threat apt1 vertex', opts=onview2)) + self.eq(nodes00[0].ndef, nodes[0].ndef) - q = ''' - [ file:bytes=(file1,) :sha256=$sha256 ] - spin | - yield $lib.gen.fileBytesBySha256($sha256) - ''' - nodes = await core.nodes(q, opts=opts) - self.len(1, nodes) - self.eq(nodes[0].repr(), s_common.guid(('file1',))) + # gen.risk.tool.software - with self.raises(s_exc.BadTypeValu): - await core.callStorm('$lib.gen.fileBytesBySha256(newp)', opts=opts) + self.len(0, await core.nodes('gen.risk.tool.software blackcat (null) --try')) + self.len(0, await core.nodes('gen.risk.tool.software (null) vertex --try')) + await self.asyncraises(s_exc.BadTypeValu, core.nodes('gen.risk.tool.software (null) vertex')) + await self.asyncraises(s_exc.BadTypeValu, core.nodes('gen.risk.tool.software blackcat (null)')) - q = 'return($lib.gen.fileBytesBySha256(newp, try=$lib.true))' - self.none(await core.callStorm(q, opts=opts)) + self.len(1, nodes00 := await core.nodes('gen.risk.tool.software blackcat vertex | [ :names+=alphv ]')) + self.len(1, nodes01 := await core.nodes('gen.risk.tool.software alphv vertex')) + self.eq('blackcat', nodes00[0].get('name')) + self.eq(['alphv'], nodes00[0].get('names')) + self.eq('vertex', nodes00[0].get('reporter:name')) + self.nn(nodes00[0].get('reporter')) + self.eq(nodes00[0].ndef, nodes01[0].ndef) + self.eq(nodes00[0].getProps(), nodes01[0].getProps()) - async def test_stormlib_gen_inetTlsServerCert(self): + self.len(1, nodes := await core.nodes('risk:tool:software:name=blackcat :reporter -> ou:org')) + self.eq(vtxguid, nodes[0].ndef[1]) - async with self.getTestCore() as core: - sha256 = s_common.buid().hex() - opts = {'vars': {'sha256': sha256}} + self.len(1, nodes := await core.nodes('gen.risk.tool.software blackcat otherorg')) + self.ne(nodes00[0].ndef[1], nodes[0].ndef[1]) + self.eq('blackcat', nodes[0].get('name')) + self.none(nodes[0].get('names')) + self.eq('otherorg', nodes[0].get('reporter:name')) + self.nn(nodes[0].get('reporter')) - q = ''' - $server = {[ inet:server="1.2.3.4:443" ]} - yield $lib.gen.inetTlsServerCertByServerAndSha256($server, $sha256) - ''' - nodes = await core.nodes(q, opts=opts) - self.len(1, nodes) - self.eq(nodes[0].get('server'), 'tcp://1.2.3.4:443') - cert = nodes[0].get('cert') - self.nn(cert) + self.len(1, nodes := await core.nodes('gen.risk.tool.software blackcat vertex', opts=onview2)) + self.eq(nodes00[0].ndef, nodes[0].ndef) - nodes = await core.nodes('crypto:x509:cert:sha256=$sha256', opts=opts) - self.len(1, nodes) - self.eq(nodes[0].repr(), cert) + # gen.risk.vuln - with self.raises(s_exc.BadTypeValu): - await core.callStorm('$lib.gen.inetTlsServerCertByServerAndSha256(newp, $sha256)', opts=opts) + self.len(0, await core.nodes('gen.risk.vuln newp (null) --try')) + self.len(0, await core.nodes('gen.risk.vuln (null) vertex --try')) + await self.asyncraises(s_exc.BadTypeValu, core.nodes('gen.risk.vuln (null) vertex')) + await self.asyncraises(s_exc.BadTypeValu, core.nodes('gen.risk.vuln newp (null)')) - q = 'return($lib.gen.inetTlsServerCertByServerAndSha256(newp, $sha256, try=$lib.true))' - self.none(await core.callStorm(q, opts=opts)) + self.len(1, nodes00 := await core.nodes('gen.risk.vuln cve-2024-0123 vertex')) + self.len(1, nodes01 := await core.nodes('gen.risk.vuln cve-2024-0123 vertex')) + self.eq('CVE-2024-0123', nodes00[0].get('id')) + self.eq('vertex', nodes00[0].get('reporter:name')) + self.nn(nodes00[0].get('reporter')) + self.eq(nodes00[0].ndef, nodes01[0].ndef) + self.eq(nodes00[0].getProps(), nodes01[0].getProps()) - async def test_stormlib_gen_cryptoX509Cert(self): + self.len(1, nodes := await core.nodes('it:sec:cve=cve-2024-0123 -> risk:vuln:id :reporter -> ou:org')) + self.eq(vtxguid, nodes[0].ndef[1]) - async with self.getTestCore() as core: + self.len(1, nodes := await core.nodes('gen.risk.vuln cve-2024-0123 otherorg')) + self.ne(nodes00[0].ndef[1], nodes[0].ndef[1]) + self.eq('CVE-2024-0123', nodes[0].get('id')) + self.eq('otherorg', nodes[0].get('reporter:name')) + self.nn(nodes[0].get('reporter')) - # Check guid generation - sha256 = s_common.buid().hex() - opts = {'vars': {'sha256': sha256}} - nodes = await core.nodes('yield $lib.gen.cryptoX509CertBySha256($sha256)', opts=opts) - self.len(1, nodes) - self.eq(nodes[0].get('sha256'), sha256) - - # Check invalid values, no try - with self.raises(s_exc.BadTypeValu): - await core.callStorm('$lib.gen.cryptoX509CertBySha256(newp)') - - # Check invalid values, with try - self.none(await core.callStorm('return($lib.gen.cryptoX509CertBySha256(newp, try=$lib.true))')) - - # Check node matching with same sha256 values - sha256 = s_common.buid().hex() - opts = {'vars': {'sha256': sha256}} - nodes = await core.nodes('[crypto:x509:cert=* :sha256=$sha256]', opts=opts) - self.len(1, nodes) - self.eq(nodes[0].get('sha256'), sha256) - self.ne(nodes[0].repr(), s_common.guid(sha256)) - crypto = nodes[0].repr() - - # Check node matching, crypto:x509:cert -> file with matching sha256 - sha256 = s_common.buid().hex() - opts = {'vars': {'sha256': sha256}} - nodes = await core.nodes('[crypto:x509:cert=* :file={[ file:bytes=({"sha256": $sha256}) ]} ]', opts=opts) - self.len(1, nodes) - self.none(nodes[0].get('sha256')) - crypto = nodes[0].repr() - - nodes = await core.nodes('yield $lib.gen.cryptoX509CertBySha256($sha256)', opts=opts) - self.len(1, nodes) + self.len(1, nodes := await core.nodes('gen.risk.vuln cve-2024-0123 vertex', opts=onview2)) + self.eq(nodes00[0].ndef, nodes[0].ndef) diff --git a/synapse/tests/test_model_entity.py b/synapse/tests/test_model_entity.py index 950c38908e..afd7cb2835 100644 --- a/synapse/tests/test_model_entity.py +++ b/synapse/tests/test_model_entity.py @@ -126,7 +126,7 @@ async def test_model_entity(self): :desc=Hehe :tag=woot.woot :sophistication=high - :reporter=$lib.gen.orgByName(vertex) + :reporter={[ ou:org=({"name": "vertex"}) ]} :reporter:name=vertex :parent={[ meta:technique=* :name=metawoot ]} ] diff --git a/synapse/tests/test_model_infotech.py b/synapse/tests/test_model_infotech.py index 1c86506d2e..f9b130c3da 100644 --- a/synapse/tests/test_model_infotech.py +++ b/synapse/tests/test_model_infotech.py @@ -513,7 +513,7 @@ async def test_it_forms_simple(self): self.len(1, await core.nodes('it:log:event :service:account -> inet:service:account')) self.len(1, await core.nodes('it:log:event :service:platform -> inet:service:platform')) - nodes = await core.nodes('it:host | limit 1 | [ :keyboard:layout=qwerty :keyboard:language=$lib.gen.langByCode(en.us) ]') + nodes = await core.nodes('it:host | limit 1 | [ :keyboard:layout=qwerty :keyboard:language={[ lang:language=({"code": "en.us"}) ]} ]') self.len(1, nodes) self.nn(nodes[0].get('keyboard:language')) self.len(1, await core.nodes('it:host:keyboard:layout=QWERTY')) diff --git a/synapse/tests/test_model_language.py b/synapse/tests/test_model_language.py index 5e37334b28..310c5d5083 100644 --- a/synapse/tests/test_model_language.py +++ b/synapse/tests/test_model_language.py @@ -30,10 +30,6 @@ async def test_model_language(self): self.len(1, await core.nodes('lang:translation -> it:software')) self.len(2, await core.nodes('lang:translation -> lang:language')) - self.none(await core.callStorm('return($lib.gen.langByCode(neeeeewp, try=$lib.true))')) - with self.raises(s_exc.BadTypeValu): - await core.callStorm('return($lib.gen.langByCode(neeeeewp))') - nodes = await core.nodes('[ lang:phrase="For The People" ]') self.len(1, nodes) self.eq('For The People', nodes[0].repr()) diff --git a/synapse/tests/test_model_transport.py b/synapse/tests/test_model_transport.py index 77084ef470..b7e56628aa 100644 --- a/synapse/tests/test_model_transport.py +++ b/synapse/tests/test_model_transport.py @@ -136,6 +136,13 @@ async def test_model_transport(self): nodes = await core.nodes(''' $regid = $lib.guid() + $contact = {[ + entity:contact=({ + "type": "us.va.dmv", + "email": "visi@vertex.link", + }) + ]} + [ transport:land:registration=$regid :id=zeroday @@ -144,7 +151,7 @@ async def test_model_transport(self): :issuer={gen.ou.org "virginia dmv"} :issuer:name="virginia dmv" - :contact={gen.ps.contact.email us.va.dmv visi@vertex.link} + :contact=$contact :vehicle={[ transport:land:vehicle=* :serial=V-31337 @@ -152,15 +159,15 @@ async def test_model_transport(self): :model=elise :registration=$regid :type=car - :owner={gen.ps.contact.email us.va.dmv visi@vertex.link} + :owner=$contact ]} :license={[ transport:land:license=* :id=V-31337 - :contact={gen.ps.contact.email us.va.dmv visi@vertex.link} + :contact=$contact :issued=20221217 :expires=20251217 - :issuer={gen.ou.org "virginia dmv"} + :issuer={[ ou:org=({"name": "virginia dmv"}) ]} :issuer:name="virginia dmv" ]} ]