escape domain regexes #141
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -124,10 +124,10 @@ def build_generic_purl(uri): | |
|
|
||
|
|
||
| @purl_router.route( | ||
| r"https?://registry\.npmjs\.(com|org)/.*", | ||
| r"https?://registry\.yarnpkg\.com/.*", | ||
| r"https?://(www\.)?npmjs\.(com|org)/package.*", | ||
| r"https?://(www\.)?yarnpkg\.com/package.*", | ||
| ) | ||
| def build_npm_purl(uri): | ||
| # npm URLs are difficult to disambiguate with regex | ||
|
|
@@ -221,9 +221,9 @@ def build_npm_web_purl(uri): | |
|
|
||
|
|
||
| @purl_router.route( | ||
| r"https?://repo1\.maven\.org/maven2/.*", | ||
| r"https?://central\.maven\.org/maven2/.*", | ||
| r"maven-index://repo1\.maven\.org/.*", | ||
| ) | ||
| def build_maven_purl(uri): | ||
| path = unquote_plus(urlparse(uri).path) | ||
|
|
@@ -253,15 +253,26 @@ def build_maven_purl(uri): | |
|
|
||
| qualifiers["classifier"] = classifier.strip("-") | ||
|
|
||
| valid_types = ( | ||
| "aar", | ||
| "ear", | ||
| "mar", | ||
| "pom", | ||
| "rar", | ||
| "rpm", | ||
| "sar", | ||
| "tar.gz", | ||
| "war", | ||
| "zip", | ||
| ) | ||
| if extension in valid_types: | ||
| qualifiers["type"] = extension | ||
|
|
||
| return PackageURL("maven", namespace, name, version, qualifiers) | ||
|
|
||
|
|
||
| # https://rubygems.org/gems/i18n-js-3.0.11.gem | ||
| @purl_router.route(r"https?://rubygems\.org/(downloads|gems)/.*") | ||
| def build_rubygems_purl(uri): | ||
| # We use a more general route pattern instead of using `rubygems_pattern` | ||
| # below by itself because we want to capture all rubygems download URLs, | ||
|
|
@@ -272,7 +283,7 @@ def build_rubygems_purl(uri): | |
| # https://rubygems.org/downloads/jwt-0.1.8.gem | ||
| # https://rubygems.org/gems/i18n-js-3.0.11.gem | ||
| rubygems_pattern = ( | ||
| r"^https?://rubygems\.org/(downloads|gems)/(?P<name>.+)-(?P<version>.+)(\.gem)$" | ||
| ) | ||
| return purl_from_pattern("gem", rubygems_pattern, uri) | ||
|
|
||
|
|
@@ -296,7 +307,7 @@ def build_cran_purl(uri): | |
| # https://pypi.python.org/packages/2.6/t/threadpool/threadpool-1.2.7-py2.6.egg | ||
| # https://pypi.python.org/packages/any/s/setuptools/setuptools-0.6c11-1.src.rpm | ||
| # https://files.pythonhosted.org/packages/84/d8/451842a5496844bb5c7634b231a2e4caf0d867d2e25f09b840d3b07f3d4b/multi_key_dict-2.0.win32.exe | ||
| pypi_pattern = r"(?P<name>(\w\.?)+(-\w+)*)-(?P<version>.+)\.(zip|tar\.gz|tar\.bz2|tgz|egg|rpm|exe)$" | ||
|
|
||
| # This pattern can be found in the following locations: | ||
| # - wheel.wheelfile.WHEEL_INFO_RE | ||
|
|
@@ -312,8 +323,9 @@ def build_cran_purl(uri): | |
|
|
||
|
|
||
| @purl_router.route( | ||
| r"https?://pypi\.org/(packages|project)/.+", | ||
| r"https?://pypi\.python\.org/(packages|project)/.*", | ||
| r"https?://files\.pythonhosted\.org/(packages|project)/.*", | ||
| ) | ||
| def build_pypi_purl(uri): | ||
| path = unquote_plus(urlparse(uri).path) | ||
|
|
@@ -357,14 +369,16 @@ def build_composer_purl(uri): | |
|
|
||
| # http://nuget.org/packages/EntityFramework/4.2.0.0 | ||
| # https://www.nuget.org/api/v2/package/Newtonsoft.Json/11.0.1 | ||
| nuget_www_pattern = ( | ||
| r"^https?://(www\.)?nuget\.org/(api/v2/)?packages?/(?P<name>.+)/(?P<version>.+)$" | ||
| ) | ||
|
|
||
| register_pattern("nuget", nuget_www_pattern) | ||
|
|
||
|
|
||
| # https://api.nuget.org/v3-flatcontainer/newtonsoft.json/10.0.1/newtonsoft.json.10.0.1.nupkg | ||
| nuget_api_pattern = ( | ||
| r"^https?://api\.nuget\.org/v3-flatcontainer/" | ||
| r"(?P<name>.+)/" | ||
| r"(?P<version>.+)/" | ||
| r".*(nupkg)$" # ends with "nupkg" | ||
|
|
@@ -378,7 +392,7 @@ def build_composer_purl(uri): | |
| # https://sourceforge.net/projects/ventoy/files/v1.0.96/Ventoy%201.0.96%20release%20source%20code.tar.gz/download | ||
| # https://sourceforge.net/projects/geoserver/files/GeoServer/2.23.4/geoserver-2.23.4-war.zip/download | ||
| sourceforge_download_pattern = ( | ||
| r"^https?://((master|iweb)\.dl\.)?sourceforge\.net/projects/" | ||
| r"(?P<name>.+)/" | ||
| r"files/" | ||
| r"(?i:(?P=name)/)?" # optional case-insensitive name segment repeated | ||
|
|
@@ -392,7 +406,7 @@ def build_composer_purl(uri): | |
|
|
||
| # https://sourceforge.net/projects/spacesniffer/files/spacesniffer_1_3_0_2.zip/download | ||
| sourceforge_download_pattern_bis = ( | ||
| r"^https?://((master|iweb)\.dl\.)?sourceforge\.net/projects/" | ||
| r"(?P<name>.+)/" | ||
| r"files/" | ||
| r"(?i:(?P=name))_*(?P<version>[0-9_]+).*" | ||
|
|
@@ -402,7 +416,7 @@ def build_composer_purl(uri): | |
| register_pattern("sourceforge", sourceforge_download_pattern_bis) | ||
|
|
||
|
|
||
| @purl_router.route(r"https?://((master|iweb)\.dl\.)?sourceforge\.net/projects?/.*") | ||
| def build_sourceforge_purl(uri): | ||
| # We use a more general route pattern instead of using `sourceforge_pattern` | ||
| # below by itself because we want to capture all sourceforge download URLs, | ||
|
|
@@ -412,7 +426,7 @@ def build_sourceforge_purl(uri): | |
|
|
||
| # http://master.dl.sourceforge.net/project/libpng/zlib/1.2.3/zlib-1.2.3.tar.bz2 | ||
| sourceforge_pattern = ( | ||
| r"^https?://((master|iweb)\.dl\.)?sourceforge\.net/projects?/" | ||
| r"(?P<namespace>([^/]+))/" # do not allow more "/" segments | ||
| r"(OldFiles/)?" | ||
| r"(?P<name>.+)/" | ||
|
|
@@ -437,27 +451,29 @@ def build_sourceforge_purl(uri): | |
| if remaining_uri_path_segments: | ||
| project_name = remaining_uri_path_segments[0] # aloyscore | ||
| sourceforge_purl = PackageURL( | ||
| type="sourceforge", | ||
| name=project_name, | ||
| qualifiers={"download_url": uri}, | ||
| ) | ||
| return sourceforge_purl | ||
|
|
||
|
|
||
| # https://crates.io/api/v1/crates/rand/0.7.2/download | ||
| cargo_pattern = r"^https?://crates\.io/api/v1/crates/(?P<name>.+)/(?P<version>.+)(\/download)$" | ||
|
|
||
| register_pattern("cargo", cargo_pattern) | ||
|
|
||
|
|
||
| # https://raw.githubusercontent.com/volatilityfoundation/dwarf2json/master/LICENSE.txt | ||
| github_raw_content_pattern = ( | ||
| r"https?://raw\.githubusercontent\.com/(?P<namespace>[^/]+)/(?P<name>[^/]+)/" | ||
| r"(?P<version>[^/]+)/(?P<subpath>.*)$" | ||
| ) | ||
|
|
||
| register_pattern("github", github_raw_content_pattern) | ||
|
|
||
|
|
||
| @purl_router.route(r"https?://api\.github\.com/repos/.*") | ||
| def build_github_api_purl(url): | ||
| """ | ||
| Return a PackageURL object from GitHub API `url`. | ||
|
|
@@ -488,54 +504,54 @@ def build_github_api_purl(url): | |
| # https://codeload.github.com/nexB/scancode-toolkit/tar.gz/v3.1.1 | ||
| # https://codeload.github.com/berngp/grails-rest/zip/release/0.7 | ||
| github_codeload_pattern = ( | ||
| r"https?://codeload\.github\.com/(?P<namespace>.+)/(?P<name>.+)/" | ||
| r"(zip|tar\.gz|tar\.bz2|tgz)/([^/]*/)*" | ||
|
There was a problem hiding this comment. Reason for this change from https://owasp.org/www-community/attacks/Regular_expression_Denial_of_Service_-_ReDoS |
||
| r"(?P<version_prefix>v|V?)(?P<version>.+)$" | ||
| ) | ||
|
|
||
| register_pattern("github", github_codeload_pattern) | ||
|
|
||
|
|
||
| @purl_router.route(r"https?://github\.com/.*") | ||
| def build_github_purl(url): | ||
| """ | ||
| Return a PackageURL object from GitHub `url`. | ||
| """ | ||
|
|
||
| # https://github.com/nexB/scancode-toolkit/archive/v3.1.1.zip | ||
| archive_pattern = ( | ||
| r"https?://github\.com/(?P<namespace>.+)/(?P<name>.+)" | ||
| r"/archive/([^/]*/)*" | ||
| r"((?P=name)(-|_|@))?" | ||
| r"(?P<version_prefix>v|V?)(?P<version>.+)\.(zip|tar\.gz|tar\.bz2|tgz)" | ||
| ) | ||
|
|
||
| # https://github.com/downloads/mozilla/rhino/rhino1_7R4.zip | ||
| download_pattern = ( | ||
| r"https?://github\.com/downloads/(?P<namespace>.+)/(?P<name>.+)/" | ||
| r"((?P=name)(-|@)?)?" | ||
| r"(?P<version_prefix>v|V?)(?P<version>.+)\.(zip|tar\.gz|tar\.bz2|tgz)" | ||
|
There was a problem hiding this comment. This also fixes a bug with tgz files, since it would previously match
due to the double unescaped See for clarification: https://regex101.com/r/luaRIy/1 |
||
| ) | ||
|
|
||
| # https://github.com/pypa/get-virtualenv/raw/20.0.31/public/virtualenv.pyz | ||
| raw_pattern = ( | ||
| r"https?://github\.com/(?P<namespace>.+)/(?P<name>.+)" | ||
| r"/raw/(?P<version_prefix>v|V?)(?P<version>[^/]+)/(?P<subpath>.*)$" | ||
| ) | ||
|
|
||
| # https://github.com/fanf2/unifdef/blob/master/unifdef.c | ||
| blob_pattern = ( | ||
| r"https?://github\.com/(?P<namespace>.+)/(?P<name>.+)" | ||
| r"/blob/(?P<version>[^/]+)/(?P<subpath>.*)$" | ||
| ) | ||
|
|
||
| releases_download_pattern = ( | ||
| r"https?://github\.com/(?P<namespace>.+)/(?P<name>.+)" | ||
| r"/releases/download/(?P<version_prefix>v|V?)(?P<version>[^/]+)/.*$" | ||
| ) | ||
|
|
||
| # https://github.com/pombredanne/schematics.git | ||
| git_pattern = r"https?://github\.com/(?P<namespace>.+)/(?P<name>.+)\.(git)" | ||
|
|
||
| patterns = ( | ||
| archive_pattern, | ||
|
|
@@ -584,7 +600,7 @@ def build_github_purl(url): | |
| ) | ||
|
|
||
|
|
||
| @purl_router.route(r"https?://bitbucket\.org/.*") | ||
| def build_bitbucket_purl(url): | ||
| """ | ||
| Return a PackageURL object from BitBucket `url`. | ||
|
|
@@ -602,9 +618,9 @@ def build_bitbucket_purl(url): | |
| name = segments[1] | ||
|
|
||
| bitbucket_download_pattern = ( | ||
| r"https?://bitbucket\.org/" | ||
| r"(?P<namespace>.+)/(?P<name>.+)/downloads/" | ||
| r"(?P<version>.+)\.(zip|tar\.gz|tar\.bz2|tgz|exe|msi)" | ||
| ) | ||
| matches = re.search(bitbucket_download_pattern, url) | ||
|
|
||
|
|
@@ -635,7 +651,7 @@ def build_bitbucket_purl(url): | |
| ) | ||
|
|
||
|
|
||
| @purl_router.route(r"https?://gitlab\.com/(?!.*/archive/).*") | ||
| def build_gitlab_purl(url): | ||
| """ | ||
| Return a PackageURL object from Gitlab `url`. | ||
|
|
@@ -675,7 +691,7 @@ def build_gitlab_purl(url): | |
|
|
||
| # https://gitlab.com/hoppr/hoppr/-/archive/v1.11.1-dev.2/hoppr-v1.11.1-dev.2.tar.gz | ||
| gitlab_archive_pattern = ( | ||
| r"^https?://gitlab\.com/" | ||
| r"(?P<namespace>.+)/(?P<name>.+)/-/archive/(?P<version>.+)/" | ||
| r"(?P=name)-(?P=version).*" | ||
| r"[^/]$" | ||
|
|
@@ -686,7 +702,7 @@ def build_gitlab_purl(url): | |
|
|
||
| # https://hackage.haskell.org/package/cli-extras-0.2.0.0/cli-extras-0.2.0.0.tar.gz | ||
| hackage_download_pattern = ( | ||
| r"^https?://hackage\.haskell\.org/package/" | ||
| r"(?P<name>.+)-(?P<version>.+)/" | ||
| r"(?P=name)-(?P=version).*" | ||
| r"[^/]$" | ||
|
|
@@ -696,13 +712,15 @@ def build_gitlab_purl(url): | |
|
|
||
|
|
||
| # https://hackage.haskell.org/package/cli-extras-0.2.0.0/ | ||
| hackage_project_pattern = ( | ||
| r"^https?://hackage\.haskell\.org/package/(?P<name>.+)-(?P<version>[^/]+)/" | ||
| ) | ||
|
|
||
| register_pattern("hackage", hackage_project_pattern) | ||
|
|
||
|
|
||
| @purl_router.route( | ||
| r"https?://storage\.googleapis\.com/google-code-archive-downloads/v2/code\.google\.com/.*" | ||
| ) | ||
| def build_generic_google_code_archive_purl(uri): | ||
| # https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com | ||
|
|
||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
There are many subdomains ... so we cannot limit this to master or iweb