diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowBaseListener.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowBaseListener.java index 163cf1f9be..3296cc6e28 100644 --- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowBaseListener.java +++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowBaseListener.java @@ -1,4 +1,4 @@ -// Generated from org/apache/metron/profiler/client/window/generated/Window.g4 by ANTLR 4.5 +// Generated from org\apache\metron\profiler\client\window\generated\Window.g4 by ANTLR 4.5 package org.apache.metron.profiler.client.window.generated; //CHECKSTYLE:OFF diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowLexer.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowLexer.java index c10764b1bc..20c54124da 100644 --- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowLexer.java +++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowLexer.java @@ -1,4 +1,4 @@ -// Generated from org/apache/metron/profiler/client/window/generated/Window.g4 by ANTLR 4.5 +// Generated from org\apache\metron\profiler\client\window\generated\Window.g4 by ANTLR 4.5 package org.apache.metron.profiler.client.window.generated; //CHECKSTYLE:OFF diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowListener.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowListener.java index b831f31119..1c3ed5e7db 100644 --- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowListener.java +++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowListener.java @@ -1,4 +1,4 @@ -// Generated from org/apache/metron/profiler/client/window/generated/Window.g4 by ANTLR 4.5 +// Generated from org\apache\metron\profiler\client\window\generated\Window.g4 by ANTLR 4.5 package org.apache.metron.profiler.client.window.generated; //CHECKSTYLE:OFF diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowParser.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowParser.java index 415d540fd4..660114cffb 100644 --- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowParser.java +++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/generated/WindowParser.java @@ -1,4 +1,4 @@ -// Generated from org/apache/metron/profiler/client/window/generated/Window.g4 by ANTLR 4.5 +// Generated from org\apache\metron\profiler\client\window\generated\Window.g4 by ANTLR 4.5 package org.apache.metron.profiler.client.window.generated; //CHECKSTYLE:OFF diff --git a/metron-platform/metron-parsers-contrib/.gitignore b/metron-platform/metron-parsers-contrib/.gitignore new file mode 100644 index 0000000000..1efb3f3ac9 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/.gitignore @@ -0,0 +1,18 @@ +# Compiled class file +*.class + +# Log file +*.log + +# Package Files +*.jar +*.war +*.ear +*.zip +*.tar.gz +*.rar + +# Editor output +target/ +.idea/ +*.iml \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/README.md b/metron-platform/metron-parsers-contrib/README.md new file mode 100644 index 0000000000..5a23240e20 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/README.md @@ -0,0 +1,369 @@ +# Parsers for Metron + +This repository contains parsers for Metron. + +## Parser components + +### ChainParser + +![ChainParser](docs/img/chainparser.png) + +The ChainParser first converts a raw text (the bytes of a raw text) to an initial JSON object with two fields: +- `original_string`: The original string object (which is equal to the raw text). +- `timestamp`: The current timestamp. + +These fields can be overwritten by ChainLinks. The ChainParser validates the intermediate JSON objects and checks whether the `original_string` field and `timestamp` field are available after a ChainLink is executed. + +The ChainParser points to a single ChainLink which is the root of the ChainLink DAG (Directed Acyclic Graph). A ChainLink points either to another ChainLink or is the final ChainLink. The resulting JSON object is the output of the ChainParser. + +### ChainLink + +![ChainLink](docs/img/chainlink.png) + +A ChainLink has a JSON Object as input and produces (potentially a modified) JSON object as output. + +#### Identity Link + +The identity link passes on the given JSON object. No changes are made to the intermediate JSON object. + +```json +{ + "chain": ["identity"], + "parsers": { + "identity": { + "class": "org.apache.metron.parsers.contrib.links.fields.IdentityLink" + } + } +} +``` + +#### Normalize Field Link + +The normalize field link applies the normalize function to all fields. The field "Hello World" is converted to "hello_world" for example. The transformations are found in the normalize function. + +```json +{ + "chain": ["normalize_fields"], + "parsers": { + "normalize_fields": { + "class": "org.apache.metron.parsers.contrib.links.fields.NormalizeFieldLink" + } + } +} +``` + +#### Rename Link + +The rename link renames field names. The following configuration renames "field1" to "fieldX" and renames "field2" to "fieldY". The renames are specified by the "renames" argument where the keys are the original field names and the values are the new field names. + +```json +{ + "chain": ["rename_fields"], + "parsers": { + "rename_fields": { + "class": "org.apache.metron.parsers.contrib.links.fields.RenameLink", + "renames": { + "field1": "fieldX", + "field2": "fieldY" + } + } + } +} +``` + +#### Render link + +The render link converts a template to a field and substitute variables in the template. + +The template is a string and might contain references to variables by specifying the variable names between "{{" and "}}". The variables are substituted by the corresponding fields of the intermediate JSON object. The "variables" argument specifies which variables should be substituted in the template. The "output" specifies the field in which the rendered template should be stored. + +The following example substitutes variables "var1" and "var2" in the given template: + +```json +{ + "chain": ["render"], + "parsers": { + "render": { + "class": "org.apache.metron.parsers.contrib.links.fields.RenderLink", + "template": "Hello {{var1}} and {{var2}}", + "variables": ["var1", "var2"], + "output": "rendered_field" + } + } +} +``` + +#### Select link + +The select link selects a fields and stores it into a special input field which is used by all ChainLinkIO parsers. + +The following example selects "field1" and stores its value into the special input field. The field which is selected is specified by the "field" argument. + +```json +{ + "chain": ["select"], + "parsers": { + "select": { + "class": "org.apache.metron.parsers.contrib.links.fields.SelectLink", + "field": "field1" + } + } +} +``` + +#### Trim value link + +The trim value link trims whitespace for each value in the intermediate JSON object. So " hello world " is transformed into "hello world". + +```json +{ + "chain": ["trim_values"], + "parsers": { + "trim_values": { + "class": "org.apache.metron.parsers.contrib.links.fields.TrimValueLink" + } + } +} +``` + +#### Blacklist Link + +The blacklist link removes specified fields from the input. + +The following example removes "field1" and "field2" from the input. The fields are specified by the "fields" argument. + +```json +{ + "chain": ["blacklist"], + "parsers": { + "blacklist": { + "class": "org.apache.metron.parsers.contrib.links.fields.BlacklistLink", + "fields": ["field1", "field2"] + } + } +} +``` + +#### Whitelist link + +The whitelist link only allowed whitelisted fields and removes all other fields. Required fields (`original_string` and `timestamp`) are automatically whitelisted. + +The following example whitelists "field1" and "field2" so these are the only fields left in the output. The fields are specified by the "fields" argument. + +```json +{ + "chain": ["whitelist"], + "parsers": { + "trim_values": { + "class": "org.apache.metron.parsers.contrib.links.fields.WhitelistLink", + "fields": ["field1", "field2"] + } + } +} +``` + +### ChainLinkIO + +![ChainLinkIO](docs/img/chainlinkio.png) + +ChainLinkIO is a specialized version of a ChainLink which uses one input field as input and produces a JSON object as output. This JSON object is then merged with the intermediate JSON object. Before a ChainLinkIO can be executed, a field must be selected and stored in the special input field. Instead of inserting a SelectLink before every ChainLinkIO, it is also possible to specify an "input" argument. The "input" argument is automatically transformed into a RenderLink and stores the result into the special input field. Note that it is possible to parse variables in the input, for example with `{"input": "{{var1}}"}`. It is also possible to use constants as input: `{"input": "constant string"}`. If no input field is specified, `original_string` is used by default. + +#### JSON Decoder link + +The JSON Decoder link decodes JSON found in the input field. + +```json +{ + "chain": ["json_decoder"], + "parsers": { + "json_decoder": { + "class": "org.apache.metron.parsers.contrib.links.io.JSONDecoderLink" + } + } +} +``` + +#### Key-Value link + +The key-value link spits on pair-delimiters and on key-value delimiters. For example, take a look at the following string: + +`KEY1=value1|KEY2=value2|KEY3=value3` + +The pair delimiter would be `|` and the key-value delimiter is `=`. The following configuration is used for parsing the example: + +```json +{ + "chain": ["keyvalue"], + "parsers": { + "keyvalue": { + "class": "org.apache.metron.parsers.contrib.links.io.KeyValueLink", + "pair_delimiter": "|", + "key_value_delimiter": "=", + "valid_key_characters": "A-Z" + } + } +} +``` + +The "pair_delimiter" argument specifies which pair delimiter is used. The "key_value_delimiter" argument specifies the key-value delimiter and the "valid_key_characters" argument specifies the characters of which a key exists. The last argument is a substring of the regular expression for detecting keys and is required for performance issues. The value "A-Z" refers to the fact that the keys consist only of uppercase characters. + +#### Regex file link + +The Regex file link reads Regex patterns from a file (line separated) and tries to find a match. When there is a match, all named variables (like `(?\d+)`) are stored in the output JSON object. + +Instead of specifying a file to load the patterns from, it is also possible to specify the `patterns` field which should be an array of patterns. It is possible to load a file from HDFS (by specifying the `hdfs://` prefix). The file is specified in the `file` field. + +```json +{ + "chain": ["parse_regexfile"], + "parsers": { + "keyvalue": { + "class": "org.apache.metron.parsers.contrib.links.io.RegexFileLink", + "file": "hdfs://my_regex_file.txt" + } + } +} +``` + +#### Regex link + +The regex link executes a regular expression on the input field and uses the first found result for the creation of the output object. + +Take a look at the following example: + +```json +{ + "chain": ["regex"], + "parsers": { + "regex": { + "class": "org.apache.metron.parsers.contrib.links.io.RegexLink", + "pattern": "(?i)(user|username)[=:](\\w+)", + "selector": { + "username": "2" + } + } + } +} +``` + +This RegexLink searches for the pattern specified by the "pattern" field. The selector specifies the output fields. Here, the output field "username" contains the result of 2nd group of the regular expression. The "selector" argument is a mapping from output field names to the desired regular expression group which is used as value. + +#### Split link + +The split link splits the input on a given delimiter. Consider the following input: + +`value1|value2|value3` + +The split link can split on the `|` delimiter to obtain all the fields. + +Take a look at the following example: + +```json +{ + "chain": ["split"], + "parsers": { + "split": { + "class": "org.apache.metron.parsers.contrib.links.io.SplitLink", + "delimiter": "|", + "selector": { + "-1": "last_field", + "0": "first_field", + "1": "second_field" + } + } + } +} +``` + +When applied on the given example, "last_field" contains "value3", "first_field" contains "value1" and "second_field" contains "value2". The "delimiter" argument specifies the delimiter to split on and the "selector" argument is a mapping from indices to fields. Negative indices might be used to traverse the items in negative order. + +#### Timestamp link + +The timestamp link uses regular expressions to search for datetime patterns in the input. Take a look at the following example. + +```json +{ + "chain": ["parse_datetime"], + "parsers": { + "parse_datetime": { + "class": "org.apache.metron.parsers.contrib.links.io.TimestampLink", + "patterns": [ + ["([0-9]{4})-([0-9]+)-([0-9]+)T([0-9]+):([0-9]+):([0-9]+).([0-9]+)([+-]{1}[0-9]{1,2}[:]?[0-9]{2})", "yyyy MM dd HH mm ss SSSSSS Z", "newest"] + ] + } + } +} +``` + +There is one argument "patterns" which specifies all the patterns. A pattern is a tuple consisting of multiple components: + +- Regular expression. +- Matcher string. +- Ordering (optional). + +Multiple patterns might be specified. The first matching pattern is used as output. + +The regular expression is used for detecting a given datetime pattern in the input. Then, all the found groups are stored space-separated in an intermediate variable. The matcher string specifies the space-separated datetime components of the result. This is required so the timestamp parser can convert the datetime string to other formats. The ordering specifies which result must be used. The ordering `newest` is used for using the latest datetime string. + +The timestamp parser generates the following fields: + +- `datetime`: The formatted version of the datetime pattern found in the input. The format is specified in the `mapping` field. +- `mapping`: The mapping used for constructing the `datetime` field. +- `timezone_available`: Whether timezone information was found in the input. +- `original_timestamp`: The exact match which was found in the input. + +## Development + +### Creating a new ChainParser + +A ChainParser mainly consists of configuration and uses the ChainParser base class. The configuration should be created under `test/resources/your_parser_name/config.json`. Take a look at different ChainParser configuration files for inspiration. Then, a test file need to be created (`test/java/nl/qsight/parserconfig/TestYourParserName`). It should contain the following contents: + +```java +package org.apache.metron.parsers.contrib.parserconfig; + +public class TestYourParserName extends TestParser { + + @Override + public String getFolder() { + return "your_parser_name"; + } +} +``` + +The only thing that need to be changed, is the `getFolder()` method, which should point to the name of the folder which was created. + +The test will run over specified log lines. These log lines must be stored in `test/resources/your_parser_name/data_input` in which every line represents a log line. It will also need expected outputs. These are stored in `test/resources/your_parser_name/data_output`. Initially, you can put empty JSON objects for every input log line. + +Then, you can run the test. It will show the duration for each of the links it consists of: + +```text +=================================================================================== +Start ChainParser: +Epoch: 0 +Logline: 0 +Input: ... +Expected output: {...} +=================================================================================== +Link: org.apache.metron.parsers.contrib.links.io.JSONDecoderLink +Duration: 12.724098 ms +----------------------------------------------------------------------------------- +Link: org.apache.metron.parsers.contrib.links.fields.RenderLink +Duration: 0.043077 ms +----------------------------------------------------------------------------------- +Link: org.apache.metron.parsers.contrib.links.io.RegexLink +Duration: 1.275897 ms +----------------------------------------------------------------------------------- +Link: org.apache.metron.parsers.contrib.links.fields.RenameLink +Duration: 0.041026 ms +----------------------------------------------------------------------------------- +Link: org.apache.metron.parsers.contrib.links.fields.RenderLink +Duration: 0.03159 ms +----------------------------------------------------------------------------------- +Link: org.apache.metron.parsers.contrib.links.io.TimestampLink +Duration: 2.045538 ms +----------------------------------------------------------------------------------- +Parser duration: 16.161226 ms +----------------------------------------------------------------------------------- +``` + +This is useful debug information and shows the bottlenecks of your parser. \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/docs/img/chainlink.png b/metron-platform/metron-parsers-contrib/docs/img/chainlink.png new file mode 100644 index 0000000000..de4e71190a Binary files /dev/null and b/metron-platform/metron-parsers-contrib/docs/img/chainlink.png differ diff --git a/metron-platform/metron-parsers-contrib/docs/img/chainlinkio.png b/metron-platform/metron-parsers-contrib/docs/img/chainlinkio.png new file mode 100644 index 0000000000..cb1ab354bc Binary files /dev/null and b/metron-platform/metron-parsers-contrib/docs/img/chainlinkio.png differ diff --git a/metron-platform/metron-parsers-contrib/docs/img/chainparser.png b/metron-platform/metron-parsers-contrib/docs/img/chainparser.png new file mode 100644 index 0000000000..8fbb2b86d7 Binary files /dev/null and b/metron-platform/metron-parsers-contrib/docs/img/chainparser.png differ diff --git a/metron-platform/metron-parsers-contrib/pom.xml b/metron-platform/metron-parsers-contrib/pom.xml new file mode 100644 index 0000000000..c696fc4927 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/pom.xml @@ -0,0 +1,143 @@ + + + 4.0.0 + + org.apache.metron.parsers.contrib + metron-parsers-contrib + 1.0.0 + + + + 1.8 + + 0.4.1 + + 19.0 + + 2.4.3 + + + + + org.apache.metron + metron-parsers + ${metron_version} + provided + + + com.google.guava + guava + ${guava_version} + + + junit + junit + 3.8.1 + + + junit + junit + RELEASE + + + com.hubspot.jinjava + jinjava + 2.0.5 + + + joda-time + joda-time + 2.9.9 + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + ${shade_version} + + true + + + + *slf4j* + + + + + + package + + shade + + + true + uber + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + com.google + com.thirdparty.guava + + + + + + storm:storm-core:* + storm:storm-lib:* + org.slf4j.impl* + org.slf4j:slf4j-log4j* + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.5.1 + + true + ${java_version} + -Xlint:unchecked + ${java_version} + true + + + + + \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/chainlink/ChainLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/chainlink/ChainLink.java new file mode 100644 index 0000000000..d2911a9a14 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/chainlink/ChainLink.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.chainlink; + +import org.apache.metron.parsers.contrib.chainparser.ChainParser; +import org.apache.metron.parsers.contrib.chainparser.ChainParser; +import org.json.simple.JSONObject; + +import java.io.Serializable; +import java.util.Map; + +/** + * A ChainLink is an atomic unit for parsing which has a JSONObject as input and produces a JSONObject as output. + */ +public abstract class ChainLink implements Serializable { + + // The next ChainLink in the DAG (when null, there is no next ChainLink) + private ChainLink next = null; + + /** + * Method for loading the configuration. + * + * @param config The configuration to use. + */ + public void configure(Map config) {} + + /** + * This method parses the given input JSONObject and produces an updated JSONObject. + * + * @param input The JSONObject used as input. + * @return The updated JSONObject. + */ + public abstract JSONObject parse(JSONObject input); + + /** + * Get the next link of the ChainLink DAG. + * + * @return The next ChainLink. + * @see ChainParser + */ + public ChainLink getNextLink() { + return next; + } + + /** + * Set the next ChainLink in the ChainLink DAG. + * + * @param next The next ChainLink. + * @see ChainParser + */ + public void setNextLink(ChainLink next) { + assert next != this : "The next link must no equal the current link."; + this.next = next; + } + + /** + * Determines whether the DAG has a next ChainLink. + * + * @return True when there is a next ChainLink, false otherwise. + */ + public boolean hasNextLink() { + return this.next != null; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/chainlink/ChainLinkIO.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/chainlink/ChainLinkIO.java new file mode 100644 index 0000000000..54f93db650 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/chainlink/ChainLinkIO.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.chainlink; + +import org.apache.metron.parsers.contrib.common.Constants; +import org.json.simple.JSONObject; + +/** + * A special ChainLink that works with an input and one or more outputs. If the output is a JSONObject, then the output + * is automatically merged with the current state. Otherwise, a special Constants.OUTPUT_MARKER field is created + * containing the output. The type of the input should be specified when creating new classes. The field specified + * by the Constants.INPUT_MARKER is casted to the desired input type. + * + * The RenderLink is capable of transforming multiple fields into a single input. Therefore, a combination of the + * RenderLink and the ChainLinkIO is capable of transforming one or more inputs to one or more outputs and covers + * all possible usages. + * + * @see ChainLink + */ +public abstract class ChainLinkIO extends ChainLink { + + /** + * This method parses the given input JSONObject and produces an updated JSONObject. + * + * @param input The JSONObject used as input. + * @return The updated JSONObject. + */ + public abstract Object parseInputField(T input); + + /** + * This method parses the given input JSONObject and produces an updated JSONObject. + * + * @param data The JSONObject used as input. + * @return The updated JSONObject. + */ + @SuppressWarnings("unchecked") + public JSONObject parse(JSONObject data) { + String field = Constants.INPUT_MARKER; + if (!data.containsKey(field)) field = Constants.ORIGINAL_STRING; + if (!data.containsKey(field)) { + throw new IllegalStateException("Field \"" + field + "\" not found in the state."); + } + T input = (T) data.get(field); + Object outputObject = this.parseInputField(input); + if (outputObject instanceof JSONObject) { + JSONObject output = (JSONObject)outputObject; + for (Object keyObject : output.keySet()) { + String key = (String) keyObject; + data.put(key, output.get(keyObject)); + } + } else { + data.put(Constants.OUTPUT_MARKER, outputObject); + } + + // Clean up the input data + data.remove(Constants.INPUT_MARKER); + + return data; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/chainparser/ChainParser.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/chainparser/ChainParser.java new file mode 100644 index 0000000000..e7b5afc6a5 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/chainparser/ChainParser.java @@ -0,0 +1,223 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.chainparser; + +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.common.Constants; +import org.apache.metron.parsers.contrib.links.fields.IdentityLink; +import org.apache.metron.parsers.contrib.utils.ConfigUtils; +import org.apache.metron.parsers.BasicParser; +import org.apache.metron.parsers.contrib.links.fields.IdentityLink; +import org.json.simple.JSONObject; + +import java.io.UnsupportedEncodingException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.apache.metron.parsers.contrib.common.Constants.ORIGINAL_STRING; +import static org.apache.metron.parsers.contrib.common.Constants.TIMESTAMP; + +/** + * The ChainParser is a composable unit consisting of a DAG of ChainLink objects. It points to the first ChainLink of + * the ChainLink DAG and each ChainLink points towards the next ChainLink item in the DAG. + */ +public class ChainParser extends BasicParser { + + // The first ChainLink of the ChainLink DAG + private ChainLink link; + // The encoding of the messages + private String encoding; + + // Pre link hook + private Method preLinkHook = null; + private Object preLinkInvoker = null; + + // Post link hook + private Method postLinkHook = null; + private Object postLinkInvoker = null; + + /** + * Initializing the ChainParser. + */ + public ChainParser() { + this.link = new IdentityLink(); + this.setEncoding("UTF-8"); + } + + /** + * An empty initialization function which is required by the BasicParser class. + */ + @Override + public void init() { + + } + + /** + * Set the initial ChainLink of the ChainLink DAG. + * + * @param link The initial ChainLink. + */ + public void setInitialLink(ChainLink link) { + this.link = link; + } + + /** + * Get the initial ChainLink of the ChainLink DAG. The default ChainLink DAG consists of one IdentityLink which + * copies the input to the output. + * + * @return The initial ChainLink. + */ + public ChainLink getInitialLink() { + return this.link; + } + + /** + * Configuration for the ChainParser. + * + * @param map A mapping from configuration items to configuration values. + */ + @Override + @SuppressWarnings("unchecked") + public void configure(Map map) { + map = ConfigUtils.compile(map); + ChainLink linkObject = ConfigUtils.getRootLink(map); + this.setInitialLink(linkObject); + } + + public void setPreLinkHook(Method hook, Object invoker) { + this.preLinkHook = hook; + this.preLinkInvoker = invoker; + } + + public void setPostLinkHook(Method hook, Object invoker) { + this.postLinkHook = hook; + this.postLinkInvoker = invoker; + } + + private void executePreLinkHook(Object... args) { + if (this.preLinkHook == null || this.preLinkInvoker == null) return; + try { + this.preLinkHook.invoke(this.preLinkInvoker, args); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new IllegalStateException("The post link is not executable."); + } + } + + private void executePostLinkHook(Object... args) { + if (this.postLinkHook == null || this.postLinkInvoker == null) return; + try { + this.postLinkHook.invoke(this.postLinkInvoker, args); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new IllegalStateException("The post link is not executable."); + } + } + + /** + * Parse a given string represented as a list of bytes into a list containing a single JSON object. + * + * @param bytes The byte representation of the input. + * @return A list containing a single JSON object which is the parsed representation of the input. + */ + @Override + @SuppressWarnings("unchecked") + public List parse(byte[] bytes) { + // Try to decode the bytes + String decodedMessage; + try { + decodedMessage = new String(bytes, this.getEncoding()); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException("Unsupported encoding."); + } + + // Put the bytes conversion and current date into an initial JSON object + JSONObject state = new JSONObject(); + Instant instant = Instant.now(); + long timestamp = instant.toEpochMilli(); + state.put(ORIGINAL_STRING, decodedMessage); + state.put(TIMESTAMP, timestamp); + + // Execute the parser DAG (Directed Acyclic Graph) + ChainLink link = this.link; + + this.executePreLinkHook(link); + state = this.executeLink(link, state); + this.executePostLinkHook(link); + + // Iterate through the DAG until the end has reached + while (link.hasNextLink()) { + link = link.getNextLink(); + this.executePreLinkHook(link); + state = this.executeLink(link, state); + this.executePostLinkHook(link); + } + + // Clean up the output marker + if (state.containsKey(Constants.OUTPUT_MARKER)) { + state.remove(Constants.OUTPUT_MARKER); + } + + // Create a list containing a single item + List resultSet = new ArrayList<>(); + resultSet.add(state); + + // Return the result + return resultSet; + } + + /** + * Execute a ChainLink and return the result. + * + * @param link ChainLink to execute. + * @param state Input data. + * @return The output of the ChainLink. + */ + private JSONObject executeLink(ChainLink link, JSONObject state) { + String linkName = link.getClass().getCanonicalName(); + + // Execute the link + state = link.parse(state); + + // Check the validness of the state after executing the link + if (!state.containsKey(ORIGINAL_STRING)) + throw new IllegalStateException("The state does not contain the \"original_string\" field after executing " + + linkName + "."); + if (!state.containsKey(TIMESTAMP)) + throw new IllegalStateException("The state does not contain the \"timestamp\" field after executing " + + linkName + "."); + + // Clean up the input field when it still exists + if (state.containsKey(Constants.INPUT_MARKER)) { + state.remove(Constants.INPUT_MARKER); + } + + return state; + } + + public String getEncoding() { + return encoding; + } + + public void setEncoding(String encoding) { + this.encoding = encoding; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/common/Constants.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/common/Constants.java new file mode 100644 index 0000000000..187b9ac3fc --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/common/Constants.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.common; + +public class Constants { + + public static String INPUT_MARKER = ""; + public static String OUTPUT_MARKER = ""; + public static String TIMESTAMP = "timestamp"; + public static String ORIGINAL_STRING = "original_string"; + public static String AUTOGENERATED_LINK = "_AUTOLINK_"; + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/BlacklistLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/BlacklistLink.java new file mode 100644 index 0000000000..50de6618d5 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/BlacklistLink.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.fields; + +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.json.simple.JSONObject; + +import java.util.List; +import java.util.Map; + +/** + * A link which blacklists keys. + */ +public class BlacklistLink extends ChainLink { + + private List fields; + + public void configure(Map config) { + this.fields = null; + if (config.containsKey("fields")) { + assert config.get("fields") instanceof List; + this.setFields((List) config.get("fields")); + } + } + + /** + * Get the fields. + * + * @return The fields. + */ + public List getFields() { + return fields; + } + + /** + * Set the fields. + * + * @param fields The fields which will be blacklisted. + */ + public void setFields(List fields) { + this.fields = fields; + } + + /** + * Whitelist all the specified fields and filter out fields not in the list. + * + * @param data Input data. + * @return Data with only whitelisted fields. + */ + @Override + @SuppressWarnings("unchecked") + public JSONObject parse(JSONObject data) { + if (this.getFields() == null) throw new IllegalStateException("The blacklisted fields should be specified."); + JSONObject result = new JSONObject(); + + for (Object keyObject : data.keySet()) { + if (keyObject instanceof String) { + String key = (String) keyObject; + if (!this.getFields().contains(key)) { + result.put(keyObject, data.get(keyObject)); + } + } + } + + return result; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/IdentityLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/IdentityLink.java new file mode 100644 index 0000000000..538a1a3908 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/IdentityLink.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.fields; + +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.json.simple.JSONObject; + +/** + * The IdentityLink returns a copy of the input as output. + */ +public class IdentityLink extends ChainLink { + + /** + * Parse a JSONObject using the IdentityLink class. + * + * @param input The JSONObject used as input. + * @return The same JSONObject. + */ + @Override + public JSONObject parse(JSONObject input) { + return input; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/NormalizeFieldLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/NormalizeFieldLink.java new file mode 100644 index 0000000000..b259587bcd --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/NormalizeFieldLink.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.fields; + +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.utils.StringUtils; +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.utils.StringUtils; +import org.json.simple.JSONObject; + +/** + * A link for normalizing field names. + */ +public class NormalizeFieldLink extends ChainLink { + + /** + * + * + * @param data Input data. + * @return Data with only whitelisted fields. + */ + @Override + @SuppressWarnings("unchecked") + public JSONObject parse(JSONObject data) { + JSONObject result = new JSONObject(); + + for (Object keyObject : data.keySet()) { + if (keyObject instanceof String) { + String key = (String) keyObject; + String newKey = StringUtils.normalize(key); + // Remove trailing and beginning underscore + if (newKey == null || newKey.length() == 0) { + newKey = null; + } + if (result.containsKey(newKey)) throw new IllegalStateException("Duplicate normalized keys."); + result.put(newKey, data.get(keyObject)); + } + } + + return result; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/RenameLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/RenameLink.java new file mode 100644 index 0000000000..6bbb2972c5 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/RenameLink.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.fields; + +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.json.simple.JSONObject; + +import java.util.Map; + +/** + * A link which renames keys. + */ +public class RenameLink extends ChainLink { + + private Map renames; + + /** + * Get the renames. + * + * @return All the renames. + */ + public Map getRenames() { + return renames; + } + + /** + * Set the renames. + * + * @param renames The key-value pairs where the keys are to original keys and the values are the field names after + * the renames. + */ + public void setRenames(Map renames) { + this.renames = renames; + } + + @SuppressWarnings("unchecked") + public void configure(Map config) { + if (config.containsKey("rename")) { + assert config.get("rename") instanceof Map; + this.setRenames((Map) config.get("rename")); + } + } + + /** + * Rename fields using the rename rules. + * + * @param data Input data. + * @return Data with renamed keys. + */ + @Override + @SuppressWarnings("unchecked") + public JSONObject parse(JSONObject data) { + if (this.getRenames() == null || this.getRenames().size() == 0) + throw new IllegalStateException("No renames specified"); + + JSONObject store = new JSONObject(); + for (String key : this.getRenames().keySet()) { + if (data.containsKey(key)) { + store.put(key, data.get(key)); + } + } + for (String key : this.getRenames().keySet()) { + if (store.containsKey(key) && data.containsKey(key)) { + String outputKey = getRenames().get(key); + data.put(outputKey, store.get(key)); + data.remove(key); + } + } + + return data; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/RenderLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/RenderLink.java new file mode 100644 index 0000000000..cc2b8d3ad1 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/RenderLink.java @@ -0,0 +1,173 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.fields; + +import com.google.common.collect.Maps; +import com.hubspot.jinjava.Jinjava; +import com.hubspot.jinjava.interpret.RenderResult; +import com.hubspot.jinjava.lib.fn.ELFunctionDefinition; +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.json.simple.JSONObject; + +import java.io.Serializable; +import java.time.Year; +import java.util.List; +import java.util.Map; + +/** + * A link for rendering a template to a new field. + */ +public class RenderLink extends ChainLink { + // The fields and constants to concatenate + private String template; + + // The name of the field to store the output in + private String outputField; + + // The used variables + private List variables; + + /** + * Get the template. + * + * @return The template. + */ + public String getTemplate() { + return template; + } + + /** + * Set the template. + * + * @param template The Jinja template. + */ + public void setTemplate(String template) { + this.template = template; + } + + /** + * Get the output field. + * + * @return The field in which the output is stored. + */ + public String getOutputField() { + return outputField; + } + + /** + * Set the output field. + * + * @param field The field in which the output is stored. + */ + public void setOutputField(String field) { + this.outputField = field; + } + + public void setVariables(List variables) { this.variables = variables; } + + public void configure(Map config) { + this.variables = null; + if (config.containsKey("template")) { + assert config.get("template") instanceof String; + this.setTemplate((String) config.get("template")); + } + if (config.containsKey("variables")) { + assert config.get("variables") instanceof List; + this.setVariables((List) config.get("variables")); + } + if (config.containsKey("output")) { + assert config.get("output") instanceof String; + this.setOutputField((String) config.get("output")); + } + } + + /** + * Parse the JSON object. All keys of the input become available in the template. For example, if the input + * contains {"user": "me"}, then the template "{{user}}" is rendered as "me". + * + * @param data The input data. + * @return The output data in which the output field (specified by the setOutputField method) is filled with the + * rendered version of the template (specified by the setTemplate method). + */ + @Override + @SuppressWarnings("unchecked") + public JSONObject parse(JSONObject data) { + if (this.getTemplate() == null) throw new IllegalStateException("No template specified."); + if (this.getOutputField() == null) throw new IllegalStateException("No output field specified."); + + // Loop in O(n) time through the data and replace variables when found + String template = this.getTemplate(); + + boolean inVariable = false; + String variableName = ""; + String buffer = ""; + int bracketOpenCount = 0; + int bracketCloseCount = 0; + for (int i = 0; i < template.length(); i++) { + char c = template.charAt(i); + + if (c == '{') { + bracketOpenCount += 1; + } else { + bracketOpenCount = 0; + } + + if (c == '}') { + if (inVariable) { + inVariable = false; + variableName = variableName.trim(); + buffer = buffer.substring(0, buffer.length() - 2); + + // Substitute the variable + if (variableName.equals("year")) { + buffer += Year.now().toString(); + } else { + if (data.containsKey(variableName)) { + buffer += data.get(variableName); + } + } + + // Clear the variable name + variableName = ""; + } + bracketCloseCount += 1; + } else { + bracketCloseCount = 0; + } + + if (inVariable) { + variableName += c; + } else { + buffer += c; + } + + if (bracketOpenCount == 2) { + inVariable = true; + } + + if (bracketCloseCount == 2) { + buffer = buffer.substring(0, buffer.length() - 2); + } + } + + data.put(this.getOutputField(), buffer); + + // Return the result + return data; + } +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/SelectLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/SelectLink.java new file mode 100644 index 0000000000..2e1eb2b0b0 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/SelectLink.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.fields; + +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.common.Constants; +import org.json.simple.JSONObject; + +import java.util.Map; + +/** + * A link which selects a field an puts it into the input marker field. + */ +public class SelectLink extends ChainLink { + + private String field; + + /** + * Get the field to select. + * + * @return The field. + */ + public String getField() { + return field; + } + + /** + * Set the field to select. + * + * @param field The field to select. + */ + public void setField(String field) { + this.field = field; + } + + public void configure(Map config) { + if (config.containsKey("field")) { + assert config.get("field") instanceof String; + this.setField((String) config.get("field")); + } + } + + /** + * Copy the selected field to select to the input marker field. + * + * @param data Input data. + * @return Data with an additional input marker field. + */ + @Override + @SuppressWarnings("unchecked") + public JSONObject parse(JSONObject data) { + if (this.getField() == null) throw new IllegalStateException("The field to select should be specified."); + + data.put(Constants.INPUT_MARKER, data.get(this.getField())); + + return data; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/TrimValueLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/TrimValueLink.java new file mode 100644 index 0000000000..c3e580fd7c --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/TrimValueLink.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.fields; + +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.utils.StringUtils; +import org.json.simple.JSONObject; + +/** + * A link for normalizing field names. + */ +public class TrimValueLink extends ChainLink { + + /** + * + * + * @param data Input data. + * @return Data with only whitelisted fields. + */ + @Override + @SuppressWarnings("unchecked") + public JSONObject parse(JSONObject data) { + JSONObject result = new JSONObject(); + + for (Object keyObject : data.keySet()) { + Object valueObject = data.get(keyObject); + if (valueObject instanceof String) { + String value = (String) valueObject; + String newValue = value.trim(); + result.put(keyObject, newValue); + } + else { + result.put(keyObject, valueObject); + } + } + return result; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/WhitelistLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/WhitelistLink.java new file mode 100644 index 0000000000..c6e58a14be --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/fields/WhitelistLink.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.fields; + +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.json.simple.JSONObject; + +import java.util.List; +import java.util.Map; + +/** + * A link which whitelists keys. + */ +public class WhitelistLink extends ChainLink { + + private List fields; + + public void configure(Map config) { + this.fields = null; + if (config.containsKey("fields")) { + assert config.get("fields") instanceof List; + this.setFields((List) config.get("fields")); + } + } + + /** + * Get the fields. + * + * @return The fields. + */ + public List getFields() { + return fields; + } + + /** + * Set the fields. + * + * @param fields The fields which will be whitelisted. + */ + public void setFields(List fields) { + this.fields = fields; + } + + /** + * Whitelist all the specified fields and filter out fields not in the list. + * + * @param data Input data. + * @return Data with only whitelisted fields. + */ + @Override + @SuppressWarnings("unchecked") + public JSONObject parse(JSONObject data) { + if (this.getFields() == null) throw new IllegalStateException("The whitelisted fields should be specified."); + JSONObject result = new JSONObject(); + + for (Object keyObject : data.keySet()) { + if (keyObject instanceof String) { + String key = (String) keyObject; + if (this.getFields().contains(key)) { + result.put(keyObject, data.get(keyObject)); + } + } + } + + return result; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/JSONDecoderLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/JSONDecoderLink.java new file mode 100644 index 0000000000..e7b439b624 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/JSONDecoderLink.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.io; + +import org.apache.metron.parsers.contrib.chainlink.ChainLinkIO; +import org.apache.metron.parsers.contrib.chainlink.ChainLinkIO; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; + +import java.util.Map; + +/** + * A link for decoding JSON. + */ +public class JSONDecoderLink extends ChainLinkIO { + + @Override + public Object parseInputField(String input) { + JSONParser parser = new JSONParser(); + JSONObject result; + try { + result = flatten((JSONObject) parser.parse(input)); + } catch (ParseException e) { + throw new IllegalStateException("Could not parse JSON in the message."); + } + + return result; + } + + @SuppressWarnings("unchecked") + private JSONObject flatten(JSONObject original) { + JSONObject result = new JSONObject(); + for (Object key : original.keySet()) { + Object value = original.get(key); + if (value instanceof JSONObject) { + JSONObject subjson = flatten((JSONObject) value); + for (Object subitem : subjson.entrySet()) { + Map.Entry entry = (Map.Entry) subitem; + String new_key = key + "." + entry.getKey(); + result.put(new_key, entry.getValue()); + } + } else { + result.put(key, value); + } + } + return result; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/KeyValueLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/KeyValueLink.java new file mode 100644 index 0000000000..82ba919970 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/KeyValueLink.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.io; + +import org.apache.metron.parsers.contrib.chainlink.ChainLinkIO; +import org.apache.metron.parsers.contrib.utils.StringUtils; +import org.apache.metron.parsers.contrib.chainlink.ChainLinkIO; +import org.apache.metron.parsers.contrib.utils.StringUtils; +import org.json.simple.JSONObject; + +import java.util.Map; +import java.util.regex.Pattern; + +/** + * A link for splitting key-value pairs in strings. + */ +public class KeyValueLink extends ChainLinkIO { + + private String keyValueDelimiter; + private String pairDelimiter; + private String validKeyChars; + + public String getKeyValueDelimiter() { + return keyValueDelimiter; + } + + public void setKeyValueDelimiter(String keyValueDelimiter) { + this.keyValueDelimiter = keyValueDelimiter; + } + + public String getPairDelimiter() { + return pairDelimiter; + } + + public void setPairDelimiter(String pairDelimiter) { + this.pairDelimiter = pairDelimiter; + } + + public String getValidKeyChars() { + return validKeyChars; + } + + public void setValidKeyChars(String validKeyChars) { + this.validKeyChars = validKeyChars; + } + + public void configure(Map config) { + if (config.containsKey("pair_delimiter")) { + assert config.get("pair_delimiter") instanceof String; + this.setPairDelimiter((String) config.get("pair_delimiter")); + } + if (config.containsKey("key_value_delimiter")) { + assert config.get("key_value_delimiter") instanceof String; + this.setKeyValueDelimiter((String) config.get("key_value_delimiter")); + } + if (config.containsKey("valid_key_characters")) { + assert config.get("valid_key_characters") instanceof String; + this.setValidKeyChars((String) config.get("valid_key_characters")); + } + } + + @Override + @SuppressWarnings("unchecked") + public Object parseInputField(String input) { + if (this.getKeyValueDelimiter() == null) throw new IllegalStateException("The key-value delimiter is not set."); + if (this.getPairDelimiter() == null) throw new IllegalStateException("The pair delimiter is not set."); + if (this.getValidKeyChars() == null) throw new IllegalStateException("The valid key characters are not set."); + + JSONObject result = new JSONObject(); + Map pairs = StringUtils.parseKeyValuePairs(input, this.getKeyValueDelimiter(), + this.getPairDelimiter(), this.getValidKeyChars()); + for (String key : pairs.keySet()) { + if (key != null && !key.equals("null")) { + result.put(key, pairs.get(key)); + } + } + return result; + } +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/RegexFileLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/RegexFileLink.java new file mode 100644 index 0000000000..48ab331f83 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/RegexFileLink.java @@ -0,0 +1,147 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.io; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.metron.parsers.contrib.chainlink.ChainLinkIO; +import org.apache.hadoop.fs.Path; +import org.json.simple.JSONObject; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A link for reading a file contatining Regex patterns and parsing the variables when a match is found. + */ +public class RegexFileLink extends ChainLinkIO { + + private List patterns; + + public void configure(Map config) { + if (config.containsKey("file")) { + assert config.get("file") instanceof String; + this.readFile((String) config.get("file")); + } + if (config.containsKey("patterns")) { + assert config.get("patterns") instanceof List; + this.setPatterns((List) config.get("patterns")); + } + } + + + @Override + public Object parseInputField(String input) { + JSONObject result = new JSONObject(); + + for (String pattern : this.patterns) { + JSONObject variables = extractVariables(input, pattern); + if (variables.keySet().size() > 0) { + return variables; + } + } + + return result; + } + + public void setPatterns(List patterns) { + this.patterns = patterns; + } + + public List getPatterns() { return this.patterns; } + + @SuppressWarnings("unchecked") + public JSONObject extractVariables(String logline, String pattern) { + JSONObject vars = new JSONObject(); + + // Skip some lines + if (pattern.startsWith("[")) return new JSONObject(); + if (pattern.trim().length() == 0) return new JSONObject(); + + // Find the non-preprocessed group names + List originalGroups = findGroups(pattern, "\\?\\<([^>]+)\\>"); + + // Find all group names + List groups = findGroups(pattern, "\\?\\<([a-zA-Z0-9]+)\\>"); + + // Find all groups + Pattern regexPattern = Pattern.compile(pattern.trim(), Pattern.CASE_INSENSITIVE); + Matcher matcher = regexPattern.matcher(logline.trim()); + + if (matcher.find()) { + for (String group : groups) { + String value = matcher.group(group); + + // Find the original group name + int index = groups.indexOf(group); + String varName = originalGroups.get(index); + + vars.put(varName, value); + } + } + + // If the number of groups are not correct, return an empty JSON Object + if (vars.keySet().size() != groups.size()) return new JSONObject(); + + return vars; + } + + private List findGroups(String input, String pattern) { + // Find all group names + Pattern groupPattern = Pattern.compile(pattern); + Matcher groupMatcher = groupPattern.matcher(input); + List groups = new ArrayList<>(); + while (groupMatcher.find()) { + groups.add(groupMatcher.group(1)); + } + + return groups; + } + + public void readFile(String filepath) { + List patterns = new ArrayList<>(); + + Path path = new Path(filepath); + try { + FileSystem fs = FileSystem.get(URI.create(filepath), new Configuration()); + if (fs.exists(path)) { + InputStream inputStream = fs.open(path); + java.util.Scanner scanner = new java.util.Scanner(inputStream).useDelimiter("\n"); + while (scanner.hasNext()) { + String pattern = scanner.next(); + if (pattern.length() > 0) { + patterns.add(pattern); + } + } + } else { + throw new IllegalStateException("Could not find the file: " + filepath); + } + } catch (IOException e) { + throw new IllegalStateException("Could not load the file: " + filepath); + } + + this.setPatterns(patterns); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/RegexLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/RegexLink.java new file mode 100644 index 0000000000..0d8155b453 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/RegexLink.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.io; + +import org.apache.metron.parsers.contrib.chainlink.ChainLinkIO; +import org.apache.metron.parsers.contrib.utils.StringUtils; +import org.apache.metron.parsers.contrib.chainlink.ChainLinkIO; +import org.apache.metron.parsers.contrib.utils.StringUtils; +import org.json.simple.JSONObject; + +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class RegexLink extends ChainLinkIO { + + private Map selector; + private String pattern; + + @SuppressWarnings("unchecked") + public void setSelector(Map selector) { + this.selector = selector; + } + + public Map getSelector() { + return this.selector; + } + + public String getPattern() { + return pattern; + } + + public void setPattern(String pattern) { + this.pattern = pattern; + } + + public void configure(Map config) { + if (config.containsKey("pattern")) { + assert config.get("pattern") instanceof String; + this.setPattern((String) config.get("pattern")); + } + if (config.containsKey("selector")) { + assert config.get("selector") instanceof Map; + this.setSelector((Map) config.get("selector")); + } + } + + @Override + @SuppressWarnings("unchecked") + public Object parseInputField(String input) { + Pattern pattern = Pattern.compile(this.pattern); + Matcher matcher = pattern.matcher(input); + + JSONObject result = null; + + if (matcher.find()) { + result = new JSONObject(); + + for (String selectorKey : this.selector.keySet()) { + result.put(selectorKey, ""); + Object positionObject = this.selector.get(selectorKey); + + int position = StringUtils.toInteger(positionObject); + boolean isPositionSet = StringUtils.isNumerical(positionObject); + if (!isPositionSet) throw new IllegalStateException("Position is not numerical."); + + String value = matcher.group(position); + if (value != null) result.put(selectorKey, value); + } + } + + return result; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/SplitLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/SplitLink.java new file mode 100644 index 0000000000..29f2d0cdbf --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/SplitLink.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.io; + +import org.apache.metron.parsers.contrib.chainlink.ChainLinkIO; +import org.apache.metron.parsers.contrib.utils.StringUtils; +import org.apache.metron.parsers.contrib.chainlink.ChainLinkIO; +import org.apache.metron.parsers.contrib.utils.StringUtils; +import org.json.simple.JSONObject; +import java.util.Map; +import java.util.regex.Pattern; + +/** + * A link for splitting strings. + */ +public class SplitLink extends ChainLinkIO { + + private String delimiter; + private boolean delimiterIsRegex; + private Map selector; + + public String getDelimiter() { + return delimiter; + } + + public boolean isDelimiterRegex() { + return this.delimiterIsRegex; + } + + public void setDelimiter(String delimiter) { + this.setDelimiter(delimiter, false); + } + + public void setDelimiter(String delimiter, boolean isRegex) { + this.delimiter = delimiter; + this.delimiterIsRegex = isRegex; + } + + @SuppressWarnings("unchecked") + public void setSelector(Map selector) { + this.selector = selector; + } + + public Map getSelector() { + return this.selector; + } + + public void configure(Map config) { + if (config.containsKey("delimiter")) { + assert config.get("delimiter") instanceof String; + this.setDelimiter((String) config.get("delimiter")); + } + if (config.containsKey("selector")) { + assert config.get("selector") instanceof Map; + this.setSelector((Map) config.get("selector")); + } + } + + @Override + @SuppressWarnings("unchecked") + public Object parseInputField(String input) { + if (this.getDelimiter() == null) throw new IllegalStateException("Delimiter is not set."); + if (this.getDelimiter() == null) throw new IllegalStateException("Delimiter RegEx boolean is not set."); + if (this.getSelector() == null) throw new IllegalStateException("Selector is not set."); + + String delimiter = this.getDelimiter(); + if (!this.isDelimiterRegex()) { + delimiter = Pattern.quote(delimiter); + } + + String[] parts = input.split("(" + delimiter + ")"); + + JSONObject result = new JSONObject(); + for (Object positionObject : this.selector.keySet()) { + int position = StringUtils.toInteger(positionObject); + boolean isPositionSet = StringUtils.isNumerical(positionObject); + if (!isPositionSet) throw new IllegalStateException("Position is not numerical."); + if (position < 0) { + position += parts.length; + } + if (position < parts.length) { + String positionLabel = (String) this.selector.get(positionObject); + String value = parts[position]; + result.put(positionLabel, value); + } + } + + return result; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/TimestampLink.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/TimestampLink.java new file mode 100644 index 0000000000..88540eccfb --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/links/io/TimestampLink.java @@ -0,0 +1,296 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.io; + +import org.apache.metron.parsers.contrib.chainlink.ChainLinkIO; +import org.apache.metron.parsers.contrib.chainlink.ChainLinkIO; +import org.apache.metron.parsers.contrib.common.Constants; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import org.json.simple.JSONObject; + +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.apache.metron.parsers.contrib.common.Constants.TIMESTAMP; + +public class TimestampLink extends ChainLinkIO { + + private List patterns; + private Map> predefined_patterns; + + @SuppressWarnings("unchecked") + public void configure(Map config) { + this.initPredefinedPatterns(); + + if (config.containsKey("patterns")) { + assert config.get("patterns") instanceof List; + this.setPatterns((List) config.get("patterns")); + } + } + + public void initPredefinedPatterns() { + Map> patterns = new HashMap<>(); + patterns.put("timestamp", Arrays.asList("([0-9]{10,13})", "t")); + this.setPredefinedPatterns(patterns); + } + + @SuppressWarnings("unchecked") + private void addTimestampToMessage(JSONObject message, Long timestamp) { + message.put(Constants.TIMESTAMP, timestamp); + } + + private String extractLocalTime(String regex_result, String timestamp_mapping, String mapping_format) { + DateTimeFormatter dtf = DateTimeFormat.forPattern(timestamp_mapping); + DateTimeFormatter df = DateTimeFormat.forPattern(timestamp_mapping); + DateTime temp = df.withOffsetParsed().parseDateTime(regex_result); + DateTimeZone theZone = temp.getZone(); + DateTime dt = dtf.withZone(theZone).parseDateTime(regex_result); + DateTimeFormatter fmt = DateTimeFormat.forPattern(mapping_format); + return fmt.print(dt); + } + + private long regexToTimestamp(String regex_result, String timestamp_mapping) { + try { + long result; + if (timestamp_mapping.equals("t")) { + result = Long.parseLong(regex_result); + } else { + DateTimeFormatter dtf = DateTimeFormat.forPattern(timestamp_mapping); + DateTime dt = dtf.withZoneUTC().parseDateTime(regex_result); + result = dt.getMillis(); + } + + // Convert to ms + if (String.valueOf(result).length() < 13) result *= 1000; + + return result; + } catch (Exception e) { + throw new IllegalStateException("A pattern (" + regex_result + ") was found, but could not be mapped using the following mapping: " + timestamp_mapping); + } + } + + @Override + @SuppressWarnings("unchecked") + public Object parseInputField(String input) { + JSONObject message = new JSONObject(); + + // Initiate objects + Date datetime = null; + String mapping_format = null; + DateFormat format = null; + String formatted_datetime = null; + int timezone_available = 0; + + // In the remainder of the code, a loop will run through the configured patterns in order to regex for the + // proper elements of the logline, and create output based on the regex result(s). + for (List pattern_mapping_object : this.getPatterns()) { + String regex_pattern; + String timestamp_mapping; + Object selection_index; + + // A pattern is a tuple (regex, mapping, indexing) or (predefined_pattern, indexing) where the predefined + // pattern is defined in the constructor of this class and also consists of a (regex, mapping) tuple. + if (pattern_mapping_object == null) { + throw new IllegalStateException("Pattern type not recognized."); + } + + // The pattern should consist of either 2 or 3 elements + if (pattern_mapping_object.size() == 2) { + // If 2 elements, then the pattern is a (predefined_pattern, indexing) tuple + assert pattern_mapping_object.get(0) instanceof String; + assert pattern_mapping_object.get(1) instanceof String || pattern_mapping_object.get(1) instanceof Integer; + List predefined_pattern = this.getPredefinedPatterns().get(pattern_mapping_object.get(0)); + regex_pattern = (String) predefined_pattern.get(0); + timestamp_mapping = (String) predefined_pattern.get(1); + selection_index = pattern_mapping_object.get(1); + } else if (pattern_mapping_object.size() == 3) { + // If 2 elements, then the pattern is a (regex, mapping, indexing) tuple + assert pattern_mapping_object.get(0) instanceof String; + assert pattern_mapping_object.get(1) instanceof String; + assert pattern_mapping_object.get(2) instanceof String || pattern_mapping_object.get(2) instanceof Integer; + regex_pattern = (String) pattern_mapping_object.get(0); + timestamp_mapping = (String) pattern_mapping_object.get(1); + selection_index = pattern_mapping_object.get(2); + } else { + throw new IllegalStateException("Pattern does not contain 2 or 3 elements."); + } + + // Compile the pattern and try to find matches + Pattern compiled_pattern = Pattern.compile(regex_pattern); + Matcher matcher = compiled_pattern.matcher(input); + + // Concatenate all matches (separated with spaces), so afterwards configured indexing criteria can be used + // (e.g. first, last, oldest, newest). + List results = new ArrayList<>(); + while (matcher.find()) { + StringBuilder regex_match = new StringBuilder(); + for (int group = 1; group <= matcher.groupCount(); group++) { + String item = matcher.group(group); + regex_match.append(" ").append(item); + } + if (regex_match.length() > 0) { + regex_match = new StringBuilder(regex_match.substring(1)); + } + results.add(regex_match.toString()); + } + + // Try to convert the found patterns to timestamps + List timestamps = new ArrayList<>(); + for (String result : results) { + timestamps.add(regexToTimestamp(result, timestamp_mapping)); + } + + // Order the found timestamps according to the selection criteria + int selected_timestamp = -1; + if (timestamps.size() == 0) { + // No timestamps found, continue to the next pattern + continue; + } else if (timestamps.size() == 1) { + // There is only one timestamp, so select the first element + selected_timestamp = 0; + addTimestampToMessage(message, timestamps.get(0)); + } else { + // Advanced selection criteria + if (selection_index instanceof Integer) { + // Select the (select_index)th element + int selection_index_integer = (int) selection_index; + if (selection_index_integer < results.size()) { + addTimestampToMessage(message, timestamps.get(selection_index_integer)); + selected_timestamp = selection_index_integer; + } + } else if (selection_index instanceof String) { + // If it is a string, select by newest or by oldest + if (selection_index.equals("newest")) { + // Find the newest timestamp + long max_timestamp = Collections.max(timestamps); + selected_timestamp = timestamps.indexOf(max_timestamp); + addTimestampToMessage(message, max_timestamp); + } else if (selection_index.equals("oldest")) { + // Find the oldest timestamp + long min_timestamp = Collections.min(timestamps); + selected_timestamp = timestamps.indexOf(min_timestamp); + addTimestampToMessage(message, min_timestamp); + } else { + // Unknown selection criteria (if it is a string) + throw new IllegalStateException("The selection index should be either \"newest\" or \"oldest\"."); + } + } + } + + if (selected_timestamp != -1) { + // Scenario 2: There is a timestamp which has been selected + String orig_regex = results.get(selected_timestamp); + timezone_available = 0; + if (timestamp_mapping.contains("X") || timestamp_mapping.contains("Z")) { + timezone_available = 1; + } + message.put("timezone_available", timezone_available); + + // if datetime found is an epoch timestamp, length is checked + if (timestamp_mapping.equals("t")) { + long timestamp_epoch = Long.parseLong(orig_regex); + if (String.valueOf(timestamp_epoch).length() < 13) { + timestamp_epoch *= 1000; + } + // timezone_available set to 1, in order to avoid that the fieldtransformation adds the + // probe-timezone and converts accordingly to timestamp, since any epoch timestamp is assumed to + // be UTC. + timezone_available = 1; + message.put("timezone_available", timezone_available); + + // formatted_datetime + datetime = new Date(timestamp_epoch); + mapping_format = "yyyy-MM-dd HH:mm:ss.SSS Z"; + format = new SimpleDateFormat(mapping_format); + format.setTimeZone(TimeZone.getTimeZone("UTC")); + formatted_datetime = format.format(datetime); + + // since the datetime found is no epoch timestamp, there are two possible mapping_formats that should be + // used to normalize the datetime output. + } else { + if (timezone_available == 0) { + mapping_format = "yyyy-MM-dd HH:mm:ss.SSS"; + } else { + mapping_format = "yyyy-MM-dd HH:mm:ss.SSS Z"; + } + // formatted_datetime + formatted_datetime = extractLocalTime(orig_regex, timestamp_mapping, mapping_format); + } + } + } + + if(formatted_datetime == null) { + // The script starts with (fallback) scenario 1 in which case a timestamp is not found/missing. In such a + // situation, datetime (=the variable used for fieldtransformation(s) in the config file of the parser) is set + // to current time. + datetime = new Date(); + + // First, datetime is formatted to a datetime with timezone "+0000" (in order to make sure that a timezone is + // always available during later fieldtransformation configuration in the config file of the parser). + mapping_format = "yyyy-MM-dd HH:mm:ss.SSS Z"; + format = new SimpleDateFormat(mapping_format); + format.setTimeZone(TimeZone.getTimeZone("UTC")); + formatted_datetime = format.format(datetime); + + // lastly for scenario 1, timezone_available is set to 1, so fieldtransformation configuration will not append + // the datetime field before converting it to epoch timestamp (see the config file of a parser). + timezone_available = 1; + } + + // The code above produced a (formatted) datetime, a mapping_format and a timezone_available variable, all for + // proper use of fieldtransformation(s) to produce the proper normalised epoch time. + message.put("datetime", formatted_datetime); + message.put("mapping", mapping_format); + message.put("timezone_available", timezone_available); + + // Lastly, original_timestamp will be set as a non-timezone version of the formatted(datetime). + String original_datetime_output_mapping = "yyyy-MM-dd HH:mm:ss.SSS"; + String original_timestamp; + if (original_datetime_output_mapping.equals(mapping_format)) { + original_timestamp = formatted_datetime; + } else { + original_timestamp = extractLocalTime(formatted_datetime, mapping_format, original_datetime_output_mapping); + } + message.put("original_timestamp", original_timestamp); + + return message; + } + + public void setPatterns(List patterns) { + this.patterns = patterns; + } + + public List getPatterns() { + return this.patterns; + } + + public void setPredefinedPatterns(Map> predefined_patterns) { + this.predefined_patterns = predefined_patterns; + } + + public Map> getPredefinedPatterns() { + return this.predefined_patterns; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/utils/ConfigUtils.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/utils/ConfigUtils.java new file mode 100644 index 0000000000..fd5effb8fc --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/utils/ConfigUtils.java @@ -0,0 +1,138 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.utils; + +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.common.Constants; +import org.apache.metron.parsers.contrib.links.fields.RenderLink; +import org.apache.metron.parsers.contrib.links.fields.SelectLink; +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.common.Constants; +import org.apache.metron.parsers.contrib.links.fields.RenderLink; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class ConfigUtils { + + public static Map compile(Map config) { + config = unfoldInput(config); + return config; + } + + @SuppressWarnings("unchecked") + public static Map unfoldInput(Map config) { + assert config.containsKey("chain"); + assert config.get("chain") instanceof List; + List links = (List) config.get("chain"); + + assert config.containsKey("parsers"); + assert config.get("parsers") instanceof Map; + Map linksConfig = (Map) config.get("parsers"); + + List unfoldedLinks = new ArrayList<>(); + + int autolinkIndex = 0; + for (Object link : links) { + String linkName = (String) link; + assert linksConfig.containsKey(linkName); + assert linksConfig.get(linkName) instanceof Map; + Map linkConfig = (Map) linksConfig.get(linkName); + + if (linkConfig.containsKey("input")) { + // Unfold it + String renderName = Constants.AUTOGENERATED_LINK + autolinkIndex; + autolinkIndex += 1; + unfoldedLinks.add(renderName); + Map renderConfig = new HashMap<>(); + + // Figure out which variables are being used to speed up the render link + String template = (String) linkConfig.get("input"); + List variables = new ArrayList<>(); + Pattern pattern = Pattern.compile("\\{\\{\\s*([^} |.:]+)}"); + Matcher matcher = pattern.matcher(template); + while (matcher.find()) { + String variable = matcher.group(1); + variables.add(variable); + } + + renderConfig.put("class", RenderLink.class.getName()); + renderConfig.put("template", template); + renderConfig.put("output", Constants.INPUT_MARKER); + renderConfig.put("variables", variables); + linksConfig.put(renderName, renderConfig); + + linkConfig.remove("input"); + } + + unfoldedLinks.add(linkName); + } + + config.put("chain", unfoldedLinks); + + return config; + } + + @SuppressWarnings("unchecked") + public static ChainLink getRootLink(Map config) { + assert config.containsKey("chain"); + assert config.get("chain") instanceof List; + List links = (List) config.get("chain"); + + assert config.containsKey("parsers"); + assert config.get("parsers") instanceof Map; + Map linksConfig = (Map) config.get("parsers"); + + assert links.size() > 0; + ChainLink prevLink = null; + ChainLink linkObject = null; + for (int i = links.size() - 1; i >= 0; i--) { + assert links.get(i) instanceof String; + String linkName = (String) links.get(i); + assert linksConfig.containsKey(linkName); + assert linksConfig.get(linkName) instanceof Map; + Map linkConfig = (Map) linksConfig.get(linkName); + assert linkConfig.containsKey("class"); + assert linkConfig.get("class") instanceof String; + String className = (String) linkConfig.get("class"); + try { + linkObject = (ChainLink) ChainLink.class.getClassLoader().loadClass(className).newInstance(); + } catch (InstantiationException e) { + throw new IllegalStateException("Could not instantiate the following link: " + className); + } catch (IllegalAccessException e) { + throw new IllegalStateException("Illegal access exception for link: " + className); + } catch (ClassNotFoundException e) { + throw new IllegalStateException("Class not found exception for link: " + className); + } + assert linkObject != null; + linkConfig.remove("class"); + linkObject.configure(linkConfig); + if (prevLink != null) { + linkObject.setNextLink(prevLink); + } + prevLink = linkObject; + } + assert linkObject != null; + return linkObject; + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/utils/JSONUtils.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/utils/JSONUtils.java new file mode 100644 index 0000000000..be0b989276 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/utils/JSONUtils.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.utils; + +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.type.MapType; +import org.codehaus.jackson.map.type.TypeFactory; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; + +import java.io.IOException; +import java.io.StringReader; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +public class JSONUtils { + + public static Map JSONToMap(String json) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + MapType type = mapper.getTypeFactory().constructMapType(Map.class, String.class, Object.class); + return mapper.readValue(json, type); + } + + public static Map JSONToMap(JSONObject json) throws IOException { + return JSONUtils.JSONToMap(json.toString()); + } + + public static JSONObject stringToJSON(String json) throws ParseException { + JSONParser parseJSON = new JSONParser(); + return (JSONObject) parseJSON.parse(json); + } + + public static JSONObject mapToJSON(Map data) { + return new JSONObject(data); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/utils/StringUtils.java b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/utils/StringUtils.java new file mode 100644 index 0000000000..6db0da9f05 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/main/java/org/apache/metron/parsers/contrib/utils/StringUtils.java @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.utils; + +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class StringUtils { + + /** + * Extracts (key, value) pairs from a string efficiently. + * + * @param message Message to find (key, value) pairs in. + * @param keyValueDelimiter Regular expression for the delimiter between the key and the value (for example: "="). + * @param pairDelimiters Regular expression for the delimiter between pairs (for example: " "). + * @param validKeyChars Valid characters for the keys (e.g. "a-zA-Z"). + * @return A map in which the (key, value) pairs are stored. + */ + @SuppressWarnings("unchecked") + public static Map parseKeyValuePairs(String message, String keyValueDelimiter, + String pairDelimiters, String validKeyChars) { + Map keyValues = new HashMap<>(); + StringBuilder splitRegex = new StringBuilder("(?= -999999; + return true; + } catch (Exception e) { + return false; + } + } else return number instanceof Integer; + } + + public static int toInteger(Object number) { + if (!StringUtils.isNumerical(number)) return -1; + if (number instanceof String) { + return Integer.parseInt((String) number); + } else if (number instanceof Integer) { + return (int) number; + } + return -1; + } + + public static String normalize(String key) { + String result = org.apache.commons.lang3.StringUtils.stripAccents(key); + result = result.replaceAll("[^A-Za-z_0-9]+", "_"); + result = result.replaceAll("[_]+", "_"); + result = result.replaceAll("^[_]*(.*)[_]*$", "$1"); + result = result.toLowerCase(); + return result; + } + +} \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/chainlink/TestChainLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/chainlink/TestChainLink.java new file mode 100644 index 0000000000..6140543a93 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/chainlink/TestChainLink.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.chainlink; + +import org.apache.metron.parsers.contrib.links.fields.IdentityLink; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestChainLink { + + private ChainLink link; + + @Before + public void setUp() { + this.link = new IdentityLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test(expected = AssertionError.class) + public void testSelfLink() { + // A self-link must not occur since it will result into infinite loops + this.link.setNextLink(this.link); + } + + @Test + public void testNextLink() { + ChainLink otherLink = new IdentityLink(); + this.link.setNextLink(otherLink); + assertEquals(otherLink, this.link.getNextLink()); + assertTrue(this.link.hasNextLink()); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/chainlink/TestChainLinkIO.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/chainlink/TestChainLinkIO.java new file mode 100644 index 0000000000..6c5102ae38 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/chainlink/TestChainLinkIO.java @@ -0,0 +1,152 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.chainlink; + +import org.apache.metron.parsers.contrib.chainlink.ChainLinkIO; +import org.apache.metron.parsers.contrib.common.Constants; +import org.apache.metron.parsers.contrib.common.Constants; +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * The ChainLinkIO is capable of transforming a single input field to one or more output fields. + */ +public class TestChainLinkIO { + + private AddExclamationMarkLink link_str; + private IncrementListLink link_list; + private MultiOutputLink link_multi; + + private class AddExclamationMarkLink extends ChainLinkIO { + + @Override + public Object parseInputField(String input) { + return input + "!"; + } + } + + private class IncrementListLink extends ChainLinkIO> { + + @Override + public Object parseInputField(List input) { + List newList = new ArrayList<>(); + for (int i : input) { + newList.add(i + 1); + } + return newList; + } + } + + @SuppressWarnings("unchecked") + private class MultiOutputLink extends ChainLinkIO { + + @Override + public Object parseInputField(String input) { + JSONObject result = new JSONObject(); + result.put("result_1", input + "?"); + result.put("result_2", input + "!"); + return result; + } + } + + @Before + public void setUp() { + this.link_str = new AddExclamationMarkLink(); + this.link_list = new IncrementListLink(); + this.link_multi = new MultiOutputLink(); + } + + @After + public void tearDown() { + this.link_str = null; + this.link_list = null; + this.link_multi = null; + } + + @Test + @SuppressWarnings("unchecked") + public void testStringOperations() { + JSONObject input = new JSONObject(); + input.put("var1", "test1"); + input.put("var2", "test2"); + input.put(Constants.INPUT_MARKER, "test3"); + + JSONObject output = this.link_str.parse(input); + + assertTrue(output.containsKey(Constants.OUTPUT_MARKER)); + assertTrue(output.containsKey("var1")); + assertTrue(output.containsKey("var2")); + assertEquals(3, output.size()); + assertEquals("test1", output.get("var1")); + assertEquals("test2", output.get("var2")); + assertEquals("test3!", output.get(Constants.OUTPUT_MARKER)); + } + + @Test + @SuppressWarnings("unchecked") + public void testListOperations() { + JSONObject input = new JSONObject(); + List intList = new ArrayList<>(); + intList.add(1); + intList.add(2); + intList.add(3); + input.put(Constants.INPUT_MARKER, intList); + + JSONObject output = this.link_list.parse(input); + + assertTrue(output.containsKey(Constants.OUTPUT_MARKER)); + List result = ((List) output.get(Constants.OUTPUT_MARKER)); + assertEquals(1, output.size()); + assertEquals(3, result.size()); + assertTrue(result.get(0) == 2); + assertTrue(result.get(1) == 3); + assertTrue(result.get(2) == 4); + } + + @Test + @SuppressWarnings("unchecked") + public void testMultiOutput() { + JSONObject input = new JSONObject(); + input.put(Constants.INPUT_MARKER, "test"); + + JSONObject output = this.link_multi.parse(input); + + assertFalse(output.containsKey(Constants.OUTPUT_MARKER)); + assertTrue(output.containsKey("result_1")); + assertTrue(output.containsKey("result_2")); + assertEquals(2, output.size()); + assertEquals("test?", output.get("result_1")); + assertEquals("test!", output.get("result_2")); + } + + @Test(expected = IllegalStateException.class) + public void testNoInputField() { + JSONObject input = new JSONObject(); + this.link_multi.parse(input); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/chainparser/TestChainParser.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/chainparser/TestChainParser.java new file mode 100644 index 0000000000..d62cf88f3d --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/chainparser/TestChainParser.java @@ -0,0 +1,147 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.chainparser; + +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.common.Constants; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.json.simple.JSONObject; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.metron.parsers.contrib.common.Constants.ORIGINAL_STRING; +import static org.apache.metron.parsers.contrib.common.Constants.TIMESTAMP; +import static org.junit.Assert.*; + +public class TestChainParser { + + private ChainParser parser; + + private class FieldRemoveLink extends ChainLink { + + String fieldToRemove = ""; + + @Override + public JSONObject parse(JSONObject input) { + input.remove(this.fieldToRemove); + return input; + } + } + + @Before + public void setUp() { + this.parser = new ChainParser(); + } + + @After + public void tearDown() { + this.parser = null; + } + + @Test + public void testRequiredFields() { + // Even without a specified link, the required fields should be set + byte[] exampleMessage = "example_message".getBytes(); + List resultSet = this.parser.parse(exampleMessage); + + // Should contain 1 message + assertEquals(1, resultSet.size()); + + // Therefore, it is possible to access the first element of the result + JSONObject state = resultSet.get(0); + + // Check whether the state contains the required "original_string" message + assertTrue("The parsed JSONObject should contain \"original_string\" field.", + state.containsKey(Constants.ORIGINAL_STRING)); + assertTrue("The parsed JSONObject should contain \"timestamp\" field.", + state.containsKey(Constants.TIMESTAMP)); + } + + @Test + public void testGetSetInitialLink() { + FieldRemoveLink link = new FieldRemoveLink(); + this.parser.setInitialLink(link); + ChainLink initialLink = this.parser.getInitialLink(); + assertEquals("The getter of the initial link should equal the item set by its setter.", link, initialLink); + } + + @Test(expected = IllegalStateException.class) + public void testNoOriginalString() { + // An exception should be thrown when a link removes the "original_string" field + FieldRemoveLink link = new FieldRemoveLink(); + link.fieldToRemove = Constants.ORIGINAL_STRING; + this.parser.setInitialLink(link); + this.parser.parse("".getBytes()); + } + + @Test(expected = IllegalStateException.class) + public void testNoTimestamp() { + // An exception should be thrown when a link removes the "timestamp" field + FieldRemoveLink link = new FieldRemoveLink(); + link.fieldToRemove = Constants.TIMESTAMP; + this.parser.setInitialLink(link); + this.parser.parse("".getBytes()); + } + + @Test(expected = IllegalStateException.class) + public void testIllegalEncoding() { + this.parser.setEncoding("unknown"); + this.parser.parse("".getBytes()); + } + + @Test + public void testEncodingGetterAndSetter() { + this.parser.setEncoding("encoding"); + assertEquals("encoding", this.parser.getEncoding()); + } + + @Test + public void testConfiguration() { + Map link_1_config = new HashMap<>(); + Map link_2_config = new HashMap<>(); + link_1_config.put("class", "org.apache.metron.parsers.contrib.links.fields.IdentityLink"); + link_2_config.put("class", "org.apache.metron.parsers.contrib.links.io.SplitLink"); + Map selector = new HashMap<>(); + selector.put("0", "first"); + link_2_config.put("delimiter", "|"); + link_2_config.put("selector", selector); + Map parser_config = new HashMap<>(); + Map links_config = new HashMap<>(); + links_config.put("identity", link_1_config); + links_config.put("split", link_2_config); + parser_config.put("parsers", links_config); + List chain = new ArrayList<>(); + chain.add("identity"); + chain.add("split"); + parser_config.put("chain", chain); + + this.parser.configure(parser_config); + List output = this.parser.parse("1|2|3".getBytes()); + assertTrue(output.size() == 1); + JSONObject message = output.get(0); + assertTrue(message.containsKey("first")); + assertEquals("1", message.get("first")); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestBlacklistLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestBlacklistLink.java new file mode 100644 index 0000000000..f74c909231 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestBlacklistLink.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.fields; + +import org.apache.metron.parsers.contrib.links.fields.BlacklistLink; +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestBlacklistLink { + + private BlacklistLink link; + + @Before + public void setUp() { + this.link = new BlacklistLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test + public void testGetSetFields() { + List fields = new ArrayList<>(); + fields.add("field"); + this.link.setFields(fields); + assertEquals(fields, this.link.getFields()); + } + + @Test + @SuppressWarnings("unchecked") + public void testBlacklistKeys() { + JSONObject input = new JSONObject(); + input.put("variable1", "test1"); + input.put("variable2", "test2"); + input.put("variable3", "test3"); + + List fields = new ArrayList<>(); + fields.add("variable2"); + fields.add("variable3"); + fields.add("variable4"); + this.link.setFields(fields); + + JSONObject output = this.link.parse(input); + + assertTrue(output.containsKey("variable1")); + assertEquals(1, output.size()); + } + + @Test(expected = IllegalStateException.class) + public void testNoFields() { + JSONObject input = new JSONObject(); + this.link.parse(input); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestNormalizeFieldLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestNormalizeFieldLink.java new file mode 100644 index 0000000000..563e9292ce --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestNormalizeFieldLink.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.fields; + +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestNormalizeFieldLink { + + private NormalizeFieldLink link; + + @Before + public void setUp() { + this.link = new NormalizeFieldLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test + @SuppressWarnings("unchecked") + public void testNormalizeKeys() { + JSONObject input = new JSONObject(); + input.put("Ĉool.keyCamelIPCase23TestI", "test1"); + + JSONObject result = this.link.parse(input); + assertTrue(result.containsKey("cool_keycamelipcase23testi")); + assertEquals("test1", result.get("cool_keycamelipcase23testi")); + assertTrue(result.size() == 1); + } + + @Test(expected = IllegalStateException.class) + @SuppressWarnings("unchecked") + public void testDuplicateKeys() { + // Test whether an exception is thrown when two fields are normalized to the same key which can lead to + // dangerous situations + JSONObject input = new JSONObject(); + input.put("Test", "test1"); + input.put("test", "test1"); + + this.link.parse(input); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestRenameLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestRenameLink.java new file mode 100644 index 0000000000..b18d863a1b --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestRenameLink.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.fields; + +import org.apache.metron.parsers.contrib.links.fields.RenameLink; +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestRenameLink { + + private RenameLink link; + + @Before + public void setUp() { + this.link = new RenameLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test + public void testGetSetRenames() { + Map renames = new HashMap<>(); + renames.put("from", "to"); + this.link.setRenames(renames); + assertEquals(renames, this.link.getRenames()); + } + + @Test + @SuppressWarnings("unchecked") + public void testRename() { + JSONObject input = new JSONObject(); + input.put("variable1", "test1"); + input.put("variable2", "test2"); + input.put("variable3", "test3"); + + Map renames = new HashMap<>(); + renames.put("variable1", "variable3"); + renames.put("variable2", "variable4"); + this.link.setRenames(renames); + + JSONObject output = this.link.parse(input); + + assertTrue(output.containsKey("variable3")); + assertTrue(output.containsKey("variable4")); + assertEquals(2, output.size()); + assertEquals(output.get("variable3"), "test1"); + assertEquals(output.get("variable4"), "test2"); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestRenderLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestRenderLink.java new file mode 100644 index 0000000000..77500b0de6 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestRenderLink.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.fields; + +import org.apache.metron.parsers.contrib.links.fields.RenderLink; +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.time.Year; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.*; + +public class TestRenderLink { + + private RenderLink link; + + @Before + public void setUp() { + this.link = new RenderLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test + public void testGetSetTemplate() { + String template = "test"; + this.link.setTemplate(template); + assertEquals(template, this.link.getTemplate()); + } + + @Test + public void testGetSetOutputField() { + String field = "output_field"; + this.link.setOutputField(field); + assertEquals(field, this.link.getOutputField()); + } + + @Test + @SuppressWarnings("unchecked") + public void testRenderTemplate() { + // Test whether a template is rendered correctly + JSONObject input = new JSONObject(); + input.put("variable1", "test1"); + input.put("variable2", "test2"); + + List variables = new ArrayList<>(); + variables.add("variable1"); + variables.add("variable2"); + this.link.setTemplate("{{variable1}} {{variable2}}."); + this.link.setOutputField("result"); + this.link.setVariables(variables); + JSONObject result = this.link.parse(input); + + assertTrue("The resulting object should have a \"result\" field.", result.containsKey("result")); + assertEquals("The template \"{{variable1}} {{variable2}}.\" should be rendered as " + + "\"test1 test2.\".", + "test1 test2.", result.get("result")); + } + + @Test(expected = IllegalStateException.class) + public void testNoOutputField() { + JSONObject input = new JSONObject(); + + this.link.setTemplate(""); + this.link.parse(input); + } + + @Test(expected = IllegalStateException.class) + public void testNoTemplate() { + JSONObject input = new JSONObject(); + + this.link.setOutputField("result"); + this.link.parse(input); + } + + @Test + public void testCurrentYearMethod() { + JSONObject input = new JSONObject(); + + this.link.setTemplate("{{year}}"); + this.link.setOutputField("result"); + List variables = new ArrayList<>(); + variables.add("year"); + this.link.setVariables(variables); + JSONObject result = this.link.parse(input); + + assertEquals("The year variable should be rendered as the current year.", + Year.now().toString(), result.get("result")); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestSelectLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestSelectLink.java new file mode 100644 index 0000000000..fe18ab09b9 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestSelectLink.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.fields; + +import org.apache.metron.parsers.contrib.common.Constants; +import org.apache.metron.parsers.contrib.links.fields.SelectLink; +import org.apache.metron.parsers.contrib.common.Constants; +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestSelectLink { + + private SelectLink link; + + @Before + public void setUp() { + this.link = new SelectLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test + public void testGetSetField() { + String template = "test"; + this.link.setField(template); + assertEquals(template, this.link.getField()); + } + + @Test + @SuppressWarnings("unchecked") + public void testSelectLink() { + JSONObject input = new JSONObject(); + input.put("var1", "test1"); + input.put("var2", "test2"); + + this.link.setField("var1"); + JSONObject result = this.link.parse(input); + assertEquals(3, result.size()); + assertTrue(result.containsKey("var1")); + assertTrue(result.containsKey("var2")); + assertTrue(result.containsKey(Constants.INPUT_MARKER)); + assertEquals("test1", result.get("var1")); + assertEquals("test2", result.get("var2")); + assertEquals("test1", result.get(Constants.INPUT_MARKER)); + } + + @Test(expected = IllegalStateException.class) + public void testNoField() { + JSONObject input = new JSONObject(); + this.link.parse(input); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestTrimValueLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestTrimValueLink.java new file mode 100644 index 0000000000..1ecccf1509 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestTrimValueLink.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.fields; + +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestTrimValueLink { + + private TrimValueLink link; + + @Before + public void setUp() { + this.link = new TrimValueLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test + @SuppressWarnings("unchecked") + public void testTrimValueFields() { + JSONObject input = new JSONObject(); + input.put("whitespace test", " test 1 "); + + JSONObject result = this.link.parse(input); + assertTrue(result.containsKey("whitespace test")); + assertEquals("test 1", result.get("whitespace test")); + assertTrue(result.size() == 1); + } +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestWhitelistLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestWhitelistLink.java new file mode 100644 index 0000000000..d828b8653b --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/fields/TestWhitelistLink.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.fields; + +import org.apache.metron.parsers.contrib.links.fields.WhitelistLink; +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestWhitelistLink { + + private WhitelistLink link; + + @Before + public void setUp() { + this.link = new WhitelistLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test + public void testGetSetFields() { + List fields = new ArrayList<>(); + fields.add("field"); + this.link.setFields(fields); + assertEquals(fields, this.link.getFields()); + } + + @Test + @SuppressWarnings("unchecked") + public void testWhitelistKeys() { + JSONObject input = new JSONObject(); + input.put("variable1", "test1"); + input.put("variable2", "test2"); + input.put("variable3", "test3"); + + List fields = new ArrayList<>(); + fields.add("variable2"); + fields.add("variable3"); + fields.add("variable4"); + this.link.setFields(fields); + + JSONObject output = this.link.parse(input); + + assertTrue(output.containsKey("variable2")); + assertTrue(output.containsKey("variable3")); + assertEquals(2, output.size()); + } + + @Test(expected = IllegalStateException.class) + public void testNoFields() { + JSONObject input = new JSONObject(); + this.link.parse(input); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestJSONDecodeLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestJSONDecodeLink.java new file mode 100644 index 0000000000..352803c2dc --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestJSONDecodeLink.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.io; + +import org.apache.metron.parsers.contrib.common.Constants; +import org.apache.metron.parsers.contrib.links.fields.WhitelistLink; +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestJSONDecodeLink { + + private JSONDecoderLink link; + + @Before + public void setUp() { + this.link = new JSONDecoderLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test + @SuppressWarnings("unchecked") + public void testJSONDecode() { + String input = "{\"a\": {\"b\": \"c\"}, \"d\": \"e\"}"; + + Object outputObject = this.link.parseInputField(input); + assertTrue(outputObject instanceof JSONObject); + JSONObject output = (JSONObject) outputObject; + + assertEquals(2, output.size()); + assertTrue(output.containsKey("a.b")); + assertTrue(output.containsKey("d")); + assertEquals("c", output.get("a.b")); + assertEquals("e", output.get("d")); + } + + @Test(expected = IllegalStateException.class) + public void testParseError() { + String input = "non-json"; + this.link.parseInputField(input); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestKeyValueLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestKeyValueLink.java new file mode 100644 index 0000000000..6877957aac --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestKeyValueLink.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.io; + +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestKeyValueLink { + + private KeyValueLink link; + + @Before + public void setUp() { + this.link = new KeyValueLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test + public void testKeyValueLink() { + String input = "hello=world|message=test"; + + this.link.setKeyValueDelimiter("="); + this.link.setPairDelimiter("|"); + this.link.setValidKeyChars("a-z"); + + Object outputObject = this.link.parseInputField(input); + assertTrue(outputObject instanceof JSONObject); + JSONObject output = (JSONObject) outputObject; + + assertEquals(2, output.size()); + assertTrue(output.containsKey("hello")); + assertTrue(output.containsKey("message")); + assertEquals("world", output.get("hello")); + assertEquals("test", output.get("message")); + } + + @Test(expected = IllegalStateException.class) + public void testNoKeyValueDelimiter() { + String input = "hello=world|message=test"; + this.link.setPairDelimiter("|"); + this.link.setValidKeyChars("a-z"); + this.link.parseInputField(input); + } + + @Test(expected = IllegalStateException.class) + public void testNoPairDelimiter() { + String input = "hello=world|message=test"; + this.link.setKeyValueDelimiter("="); + this.link.setValidKeyChars("a-z"); + this.link.parseInputField(input); + } + + @Test(expected = IllegalStateException.class) + public void testNoValidCharacters() { + String input = "hello=world|message=test"; + this.link.setKeyValueDelimiter("="); + this.link.setPairDelimiter("|"); + this.link.parseInputField(input); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestRegexFileLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestRegexFileLink.java new file mode 100644 index 0000000000..61f212ce6f --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestRegexFileLink.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.io; + +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestRegexFileLink { + + private RegexFileLink link; + + @Before + public void setUp() { + this.link = new RegexFileLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test + public void testRegexFileLink() { + List patterns = new ArrayList<>(); + patterns.add("NUM:(?\\d+)"); + String logline = "NUM:1234"; + this.link.setPatterns(patterns); + JSONObject output = (JSONObject) this.link.parseInputField(logline); + assertTrue(output.containsKey("number")); + assertEquals("1234", output.get("number")); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestRegexLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestRegexLink.java new file mode 100644 index 0000000000..f6b3f87c1d --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestRegexLink.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.io; + +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestRegexLink { + + private RegexLink link; + + @Before + public void setUp() { + this.link = new RegexLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test + public void testRegexLink() { + String input = "01-01-1970"; + Map selector = new HashMap<>(); + selector.put("day", "1"); + selector.put("month", "2"); + selector.put("year", "3"); + selector.put("all", "0"); + + this.link.setSelector(selector); + this.link.setPattern("([0-9]{1,2})-([0-9]{1,2})-([0-9]{4})"); + + Object outputObject = this.link.parseInputField(input); + assertTrue(outputObject instanceof JSONObject); + JSONObject output = (JSONObject) outputObject; + assertEquals(4, output.size()); + assertTrue(output.containsKey("all")); + assertTrue(output.containsKey("month")); + assertTrue(output.containsKey("year")); + assertTrue(output.containsKey("day")); + assertEquals("01", output.get("day")); + assertEquals("01", output.get("month")); + assertEquals("1970", output.get("year")); + assertEquals("01-01-1970", output.get("all")); + } + + @Test(expected = IllegalStateException.class) + public void testIllegalIndex() { + String input = "test_input"; + Map selector = new HashMap<>(); + selector.put("test_key", "illegal_index"); + this.link.setSelector(selector); + this.link.setPattern("(.*)"); + this.link.parseInputField(input); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestSplitLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestSplitLink.java new file mode 100644 index 0000000000..51a4e13d07 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestSplitLink.java @@ -0,0 +1,163 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.io; + +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestSplitLink { + + private SplitLink link; + + @Before + public void setUp() { + this.link = new SplitLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test + public void testSplitLink() { + String input = "some_header|some_message|some_footer"; + Map selector = new HashMap<>(); + selector.put("-1", "test_1"); + selector.put("1", "test_2"); + + this.link.setDelimiter("|"); + this.link.setSelector(selector); + + Object outputObject = this.link.parseInputField(input); + assertTrue(outputObject instanceof JSONObject); + JSONObject output = (JSONObject) outputObject; + assertEquals(2, output.size()); + assertTrue(output.containsKey("test_1")); + assertTrue(output.containsKey("test_2")); + assertEquals("some_footer", output.get("test_1")); + assertEquals("some_message", output.get("test_2")); + } + + @Test + public void testSplitLinkRegex() { + String input = "some_header||||some_message||some_footer"; + Map selector = new HashMap<>(); + selector.put("-1", "test_1"); + selector.put("1", "test_2"); + + this.link.setDelimiter("\\|+", true); + this.link.setSelector(selector); + + Object outputObject = this.link.parseInputField(input); + assertTrue(outputObject instanceof JSONObject); + JSONObject output = (JSONObject) outputObject; + assertEquals(2, output.size()); + assertTrue(output.containsKey("test_1")); + assertTrue(output.containsKey("test_2")); + assertEquals("some_footer", output.get("test_1")); + assertEquals("some_message", output.get("test_2")); + } + + @Test + public void testOutOfBounds() { + String input = "some_header|some_message|some_footer"; + Map selector = new HashMap<>(); + selector.put("5", "test_1"); + + this.link.setDelimiter("|"); + this.link.setSelector(selector); + + Object outputObject = this.link.parseInputField(input); + assertTrue(outputObject instanceof JSONObject); + JSONObject output = (JSONObject) outputObject; + assertEquals(0, output.size()); + } + + @Test + public void testIntegerPositions() { + String input = "some_header|some_message|some_footer"; + Map selector = new HashMap<>(); + selector.put(1, "test_1"); + + this.link.setDelimiter("|"); + this.link.setSelector(selector); + + Object outputObject = this.link.parseInputField(input); + assertTrue(outputObject instanceof JSONObject); + JSONObject output = (JSONObject) outputObject; + assertEquals(1, output.size()); + assertTrue(output.containsKey("test_1")); + assertEquals("some_message", output.get("test_1")); + } + + @Test(expected = IllegalStateException.class) + public void testIllegalIndex() { + String input = "some_header|some_message|some_footer"; + Map selector = new HashMap<>(); + selector.put("illegal_index", "test_1"); + + this.link.setDelimiter("|"); + this.link.setSelector(selector); + + this.link.parseInputField(input); + } + + @Test(expected = IllegalStateException.class) + public void testIllegalDelimiter() { + String input = "some_header|some_message|some_footer"; + Map selector = new HashMap<>(); + selector.put("0", "test_1"); + this.link.setSelector(selector); + this.link.parseInputField(input); + } + + @Test(expected = IllegalStateException.class) + public void testIllegalSelector() { + String input = "some_header|some_message|some_footer"; + this.link.setDelimiter("|"); + this.link.parseInputField(input); + } + + @Test + public void testGetSetDelimiter() { + this.link.setDelimiter("|"); + assertEquals("|", this.link.getDelimiter()); + assertEquals(false, this.link.isDelimiterRegex()); + this.link.setDelimiter(".*", true); + assertEquals(".*", this.link.getDelimiter()); + assertEquals(true, this.link.isDelimiterRegex()); + } + + @Test + public void testGetSetSelector() { + Map selector = new HashMap<>(); + selector.put("0", "test_1"); + this.link.setSelector(selector); + assertEquals(selector, this.link.getSelector()); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestTimestampLink.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestTimestampLink.java new file mode 100644 index 0000000000..d6e99e8c6d --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/links/io/TestTimestampLink.java @@ -0,0 +1,153 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.links.io; + +import org.json.simple.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestTimestampLink { + + private TimestampLink link; + + @Before + public void setUp() { + this.link = new TimestampLink(); + } + + @After + public void tearDown() { + this.link = null; + } + + @Test + @java.lang.SuppressWarnings("unchecked") + public void testTimestamp() { + String input = "1488801958"; + + List patterns = new ArrayList(); + List pattern = new ArrayList(); + pattern.add("timestamp"); + pattern.add("newest"); + patterns.add(pattern); + this.link.setPatterns(patterns); + this.link.initPredefinedPatterns(); + Object outputObject = this.link.parseInputField(input); + assertTrue(outputObject instanceof JSONObject); + JSONObject output = (JSONObject) outputObject; + + assertTrue("Output should contain a \"datetime\" field.", output.containsKey("datetime")); + assertTrue("Output should contain a \"mapping\" field.", output.containsKey("mapping")); + assertTrue("Output should contain a \"timezone_available\" field.", output.containsKey("timezone_available")); + + assertEquals("Value for \"datetime\" field is not correct.", "2017-03-06 12:05:58.000 +0000", output.get("datetime")); + assertEquals("Value for \"mapping\" field is not correct.", "yyyy-MM-dd HH:mm:ss.SSS Z", output.get("mapping")); + assertEquals("Value for \"timezone_available\" field is not correct.", 1, output.get("timezone_available")); + } + + @Test(expected = IllegalStateException.class) + @java.lang.SuppressWarnings("unchecked") + public void testIllegalInput() { + String input = "9999-99-99 99:99:99"; + + List patterns = new ArrayList(); + List pattern = new ArrayList(); + pattern.add("([0-9]{4})-([0-9]{2})-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2})"); + pattern.add("yyyy MM dd HH mm ss"); + pattern.add("oldest"); + patterns.add(pattern); + this.link.setPatterns(patterns); + this.link.parseInputField(input); + } + + @Test + @java.lang.SuppressWarnings("unchecked") + public void testDatetimeWithoutTimezone() { + String input = "2017-01-01 01:01:01"; + + List patterns = new ArrayList(); + List pattern = new ArrayList(); + pattern.add("([0-9]{4})-([0-9]{2})-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2})"); + pattern.add("yyyy MM dd HH mm ss"); + pattern.add("oldest"); + patterns.add(pattern); + this.link.setPatterns(patterns); + Object outputObject = this.link.parseInputField(input); + assertTrue(outputObject instanceof JSONObject); + JSONObject output = (JSONObject) outputObject; + + assertTrue("Output should contain a \"datetime\" field.", output.containsKey("datetime")); + assertTrue("Output should contain a \"mapping\" field.", output.containsKey("mapping")); + assertTrue("Output should contain a \"timezone_available\" field.", output.containsKey("timezone_available")); + + assertEquals("Value for \"datetime\" field is not correct.", "2017-01-01 01:01:01.000", output.get("datetime")); + assertEquals("Value for \"mapping\" field is not correct.", "yyyy-MM-dd HH:mm:ss.SSS", output.get("mapping")); + assertEquals("Value for \"timezone_available\" field is not correct.", 0, output.get("timezone_available")); + } + + @Test + @java.lang.SuppressWarnings("unchecked") + public void testSelectionCriteria() { + String input = "2017-01-01 01:01:01 | 2017-01-01 02:01:01"; + + List patterns = new ArrayList(); + List pattern = new ArrayList(); + pattern.add("([0-9]{4})-([0-9]{2})-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2})"); + pattern.add("yyyy MM dd HH mm ss"); + pattern.add("oldest"); + patterns.add(pattern); + this.link.setPatterns(patterns); + Object outputObject = this.link.parseInputField(input); + assertTrue(outputObject instanceof JSONObject); + JSONObject output = (JSONObject) outputObject; + + assertTrue("Output should contain a \"datetime\" field.", output.containsKey("datetime")); + assertTrue("Output should contain a \"mapping\" field.", output.containsKey("mapping")); + assertTrue("Output should contain a \"timezone_available\" field.", output.containsKey("timezone_available")); + + assertEquals("Value for \"datetime\" field is not correct.", "2017-01-01 01:01:01.000", output.get("datetime")); + assertEquals("Value for \"mapping\" field is not correct.", "yyyy-MM-dd HH:mm:ss.SSS", output.get("mapping")); + assertEquals("Value for \"timezone_available\" field is not correct.", 0, output.get("timezone_available")); + } + + @Test + @java.lang.SuppressWarnings("unchecked") + public void testPatternsGetterAndSetter() { + List patterns = new ArrayList(); + this.link.setPatterns(patterns); + assertEquals(patterns, this.link.getPatterns()); + } + + @Test + @java.lang.SuppressWarnings("unchecked") + public void testPredefinedPatternsGetterAndSetter() { + Map> predefinedPatterns = new HashMap(); + this.link.setPredefinedPatterns(predefinedPatterns); + assertEquals(predefinedPatterns, this.link.getPredefinedPatterns()); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestDhcp.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestDhcp.java new file mode 100644 index 0000000000..c94e24cb1f --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestDhcp.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.parserconfig; + +public class TestDhcp extends TestParser { + + @Override + public String getFolder() { + return "dhcp"; + } +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestOssec.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestOssec.java new file mode 100644 index 0000000000..a7379dcab8 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestOssec.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.parserconfig; + +public class TestOssec extends TestParser { + + @Override + public String getFolder() { + return "ossec"; + } +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestParser.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestParser.java new file mode 100644 index 0000000000..d4c02424db --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestParser.java @@ -0,0 +1,195 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.parserconfig; + +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.chainparser.ChainParser; +import org.apache.metron.parsers.contrib.utils.JSONUtils; +import org.apache.metron.parsers.contrib.chainlink.ChainLink; +import org.apache.metron.parsers.contrib.chainparser.ChainParser; +import org.apache.metron.parsers.contrib.utils.JSONUtils; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; +import org.junit.Test; +import sun.reflect.generics.reflectiveObjects.NotImplementedException; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.junit.Assert.assertEquals; + +abstract class TestParser { + + private long lastTime; + private long lastParserTime; + private long totalTime; + private long totalTimeExceptFirst; + private int runs; + + public String getFolder() { + throw new NotImplementedException(); + } + + private String readFile(String fileName) throws IOException { + try (BufferedReader br = new BufferedReader(new FileReader(fileName))) { + StringBuilder sb = new StringBuilder(); + String line = br.readLine(); + + while (line != null) { + sb.append(line); + sb.append("\n"); + line = br.readLine(); + } + return sb.toString(); + } + } + + private Map getConfig() throws IOException { + ClassLoader classLoader = getClass().getClassLoader(); + String json = this.readFile(Objects.requireNonNull(classLoader.getResource(this.getFolder() + "/config.json")).getFile()); + return JSONUtils.JSONToMap(json); + } + + private String[] getInputLoglines() throws IOException { + ClassLoader classLoader = getClass().getClassLoader(); + String data = this.readFile(Objects.requireNonNull(classLoader.getResource(this.getFolder() + "/data_input")).getFile()); + return data.split("\\r?\\n"); + } + + private String[] getOutputLoglines() throws IOException { + ClassLoader classLoader = getClass().getClassLoader(); + String data = this.readFile(Objects.requireNonNull(classLoader.getResource(this.getFolder() + "/data_output")).getFile()); + return data.split("\\r?\\n"); + } + + @Test + public void testConfigure() throws IOException { + ChainParser parser = new ChainParser(); + parser.configure(this.getConfig()); + } + + @Test + public void testInputAndOutput() throws IOException { + String inputLines[] = getInputLoglines(); + String outputLines[] = getOutputLoglines(); + assertEquals("The number of input loglines (" + inputLines.length + ") should equal the number of expected output loglines (" + outputLines.length + ").", inputLines.length, outputLines.length); + } + + public void preLinkHook(ChainLink link) { + lastTime = System.nanoTime(); + System.out.println("Link: " + link.getClass().getName()); + + } + + public void postLinkHook(ChainLink link) { + long currentTime = System.nanoTime(); + long duration = currentTime - lastTime; + this.lastParserTime += duration; + System.out.println("Duration: " + Double.toString(duration / 1000000.) + " ms"); + System.out.println("-----------------------------------------------------------------------------------"); + } + + @Test + public void testParser() throws IOException, NoSuchMethodException { + JSONParser parseJSON = new JSONParser(); + + ChainParser parser = new ChainParser(); + parser.configure(this.getConfig()); + + Method preLinkHook = this.getClass().getMethod("preLinkHook", ChainLink.class); + parser.setPreLinkHook(preLinkHook, this); + + Method postLinkHook = this.getClass().getMethod("postLinkHook", ChainLink.class); + parser.setPostLinkHook(postLinkHook, this); + + String inputLines[] = getInputLoglines(); + String outputLines[] = getOutputLoglines(); + + // By testInputAndOutput we are assured that inputLines.length == outputLines.length + runs = 0; + totalTime = 0; + totalTimeExceptFirst = 0; + + // Make sure that there are at least some amount of runs + int numEpochs = Math.max(1, (int) Math.ceil(((10. + 1.) / inputLines.length))); + + for (int epoch = 0; epoch < numEpochs; epoch++) { + for (int line = 0; line < inputLines.length; line++) { + runs += 1; + String input = inputLines[line]; + String strExpectedOutput = outputLines[line]; + System.out.println("==================================================================================="); + System.out.println("Start ChainParser:"); + System.out.println("Epoch: " + Integer.toString(epoch)); + System.out.println("Logline: " + Integer.toString(line)); + System.out.println("Input: " + input); + System.out.println("Expected output: " + strExpectedOutput); + System.out.println("==================================================================================="); + JSONObject expectedOutput; + + lastParserTime = 0; + + try { + expectedOutput = (JSONObject) parseJSON.parse(strExpectedOutput); + } catch (ParseException exception) { + exception.printStackTrace(); + throw new IllegalStateException("Line " + line + " of the output file is not a valid JSON file."); + } + + List outputMessages = parser.parse(input.getBytes()); + assertEquals("There should be exactly one message produced by the parsers.", 1, outputMessages.size()); + JSONObject output = outputMessages.get(0); + assertEquals("The output by applying the parser on line " + line + " of the data_input file does not match the expected output of line " + line + " in the data_output file.", expectedOutput.toString(), output.toString()); + + totalTime += lastParserTime; + if (line > 0) totalTimeExceptFirst += lastParserTime; + System.out.println("Parser duration: " + Double.toString(lastParserTime / 1000000.) + " ms"); + System.out.println("-----------------------------------------------------------------------------------"); + System.out.println(); + } + } + + System.out.println("==================================================================================="); + System.out.println("Overall statistics"); + System.out.println("==================================================================================="); + System.out.println("No. runs: " + Integer.toString(runs)); + System.out.println("Total duration: " + Double.toString(totalTime / 1000000.) + " ms"); + System.out.println("Avg. duration: " + Double.toString(totalTime / 1000000. / (double) runs) + " ms"); + System.out.println("-----------------------------------------------------------------------------------"); + System.out.println(); + + System.out.println("==================================================================================="); + System.out.println("Statistics (excluding first run since it initializes all classes)"); + System.out.println("==================================================================================="); + double avgDuration = totalTimeExceptFirst / 1000000. / ((double) runs - 1); + int eps = (int) (1000. / avgDuration); + System.out.println("No. runs: " + Integer.toString(runs - 1)); + System.out.println("Total duration: " + Double.toString(totalTimeExceptFirst / 1000000.) + " ms"); + System.out.println("Avg. duration: " + Double.toString(avgDuration) + " ms"); + System.out.println("Single-node EPS: " + Integer.toString(eps)); + System.out.println("-----------------------------------------------------------------------------------"); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestSuricata.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestSuricata.java new file mode 100644 index 0000000000..48d0075880 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestSuricata.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.parserconfig; + +public class TestSuricata extends TestParser { + + @Override + public String getFolder() { + return "suricata"; + } +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestYaf.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestYaf.java new file mode 100644 index 0000000000..27fd3277b2 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/parserconfig/TestYaf.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.parserconfig; + +public class TestYaf extends TestParser { + + @Override + public String getFolder() { + return "yaf"; + } +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/utils/TestConfigUtils.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/utils/TestConfigUtils.java new file mode 100644 index 0000000000..90f8caf221 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/utils/TestConfigUtils.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.utils; + +import org.apache.metron.parsers.contrib.links.fields.RenderLink; +import org.apache.metron.parsers.contrib.common.Constants; +import org.apache.metron.parsers.contrib.links.fields.RenderLink; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static junit.framework.TestCase.assertTrue; +import static org.apache.metron.parsers.contrib.common.Constants.INPUT_MARKER; +import static org.junit.Assert.assertEquals; + +public class TestConfigUtils { + + @Test + public void testUnfoldInput() throws ParseException, IOException { + String strConfig = "{\"chain\": [\"example\"], \"parsers\": {\"example\": {\"input\": \"{{var1}}\"}}}"; + JSONParser parseJSON = new JSONParser(); + JSONObject configJSON = (JSONObject) parseJSON.parse(strConfig); + Map config = JSONUtils.JSONToMap(configJSON); + Map parserConfig = ConfigUtils.unfoldInput(config); + + assertTrue(parserConfig.containsKey("chain")); + assertTrue(parserConfig.get("chain") instanceof List); + List chain = (List) parserConfig.get("chain"); + assertTrue(chain.size() == 2); + assertTrue(parserConfig.get("parsers") instanceof Map); + Map linksConfig = (Map) parserConfig.get("parsers"); + + assertTrue(chain.get(0) instanceof String); + String autolink = (String) chain.get(0); + assertTrue(linksConfig.containsKey(autolink)); + assertTrue(linksConfig.get(autolink) instanceof Map); + Map configAutolink = (Map) linksConfig.get(autolink); + + assertTrue(configAutolink.containsKey("class")); + assertTrue(configAutolink.containsKey("template")); + assertTrue(configAutolink.containsKey("output")); + + assertTrue(configAutolink.get("class") instanceof String); + assertTrue(configAutolink.get("template") instanceof String); + assertTrue(configAutolink.get("output") instanceof String); + + assertEquals(RenderLink.class.getName(), configAutolink.get("class")); + assertEquals("{{var1}}", configAutolink.get("template")); + Assert.assertEquals(Constants.INPUT_MARKER, configAutolink.get("output")); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/utils/TestJSONUtils.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/utils/TestJSONUtils.java new file mode 100644 index 0000000000..c5f76be70a --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/utils/TestJSONUtils.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.utils; + +import org.json.simple.JSONObject; +import org.json.simple.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; +import java.util.Map; + +import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertEquals; + +public class TestJSONUtils { + + @Test + public void testJSONStringToMap() throws IOException { + String json = "{\"a\": \"b\"}"; + Map result = JSONUtils.JSONToMap(json); + assertTrue(result.containsKey("a")); + assertEquals(1, result.size()); + assertTrue(result.get("a") instanceof String); + assertEquals("b", result.get("a")); + } + + @Test + public void testJSONObjectToMap() throws ParseException, IOException { + String json = "{\"a\": \"b\"}"; + JSONObject jsonObject = JSONUtils.stringToJSON(json); + Map result = JSONUtils.JSONToMap(jsonObject); + assertTrue(result.containsKey("a")); + assertEquals(1, result.size()); + assertTrue(result.get("a") instanceof String); + assertEquals("b", result.get("a")); + } + + @Test + public void testStringToJSON() throws ParseException { + String json = "{\"a\": \"b\"}"; + JSONObject jsonObject = JSONUtils.stringToJSON(json); + assertTrue(jsonObject.containsKey("a")); + assertTrue(jsonObject.get("a") instanceof String); + assertEquals(jsonObject.get("a"), "b"); + } + + @Test + public void testMapToJSON() throws IOException { + String json = "{\"a\": \"b\"}"; + Map map = JSONUtils.JSONToMap(json); + JSONObject jsonObject = JSONUtils.mapToJSON(map); + assertTrue(jsonObject.containsKey("a")); + assertTrue(jsonObject.get("a") instanceof String); + assertEquals(jsonObject.get("a"), "b"); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/utils/TestStringUtils.java b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/utils/TestStringUtils.java new file mode 100644 index 0000000000..eeba378022 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/java/org/apache/metron/parsers/contrib/utils/TestStringUtils.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.contrib.utils; + +import org.junit.Test; + +import java.util.Map; + +import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertEquals; + +// @todo add tests + +public class TestStringUtils { + + @Test + public void testParseKeyValuePairs() { + Map pairs = StringUtils.parseKeyValuePairs("hello: world, test: message", ": ", ",", "a-z"); + + assertEquals(2, pairs.size()); + assertTrue(pairs.containsKey("test")); + assertTrue(pairs.containsKey("hello")); + assertEquals("world", pairs.get("hello")); + assertEquals("message", pairs.get("test")); + } + +} diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/dhcp/config.json b/metron-platform/metron-parsers-contrib/src/test/resources/dhcp/config.json new file mode 100644 index 0000000000..8cba298ec0 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/resources/dhcp/config.json @@ -0,0 +1,72 @@ +{ + "chain": [ + "key_value_parser", + "trim_whitespaces", + "parse_datetime", + "parse_macaddress", + "parse_message_type", + "parse_hostname", + "normalize_fields", + "rename_fields" + ], + "parsers": { + "key_value_parser": { + "class": "org.apache.metron.parsers.contrib.links.io.KeyValueLink", + "pair_delimiter": "|", + "key_value_delimiter": ":", + "valid_key_characters": "a-zA-z" + }, + "trim_whitespaces": { + "class" : "org.apache.metron.parsers.contrib.links.fields.TrimValueLink" + }, + "parse_datetime": { + "class": "org.apache.metron.parsers.contrib.links.io.TimestampLink", + "patterns": [ + [ + "([0-9]{4})-([0-9]{2})-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2}).([0-9]{3})", + "yyyy MM dd HH mm ss SSS", + "newest" + ], + [ + "([0-9]{4})-([0-9]{2})-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2}).([0-9]{3}) ([+-]{1}[0-9]{1,2}[:][0-9]{2})", + "yyyy MM dd HH mm ss SSS Z", + "newest" + ] + ], + "input": "{{TIME}}" + }, + "parse_macaddress": { + "class": "org.apache.metron.parsers.contrib.links.io.RegexLink", + "pattern": "(Client-identifier: )([^\\|]++)", + "selector": { + "mac_address": "2" + }, + "input": "{{original_string}}" + }, + "parse_message_type": { + "class": "org.apache.metron.parsers.contrib.links.io.RegexLink", + "pattern": "(DHCP message type: \\d \\|)([\\w]++)", + "selector": { + "message_type": "2" + }, + "input": "{{original_string}}" + }, + "parse_hostname": { + "class": "org.apache.metron.parsers.contrib.links.io.RegexLink", + "pattern": "(Host name: )([^\\|]++)", + "selector": { + "hostname": "2" + }, + "input": "{{original_string}}" + }, + "normalize_fields": { + "class" : "org.apache.metron.parsers.contrib.links.fields.NormalizeFieldLink" + }, + "rename_fields": { + "class": "org.apache.metron.parsers.contrib.links.fields.RenameLink", + "rename": { + "ciaddr": "ip_src_addr" + } + } + } +} \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/dhcp/data_input b/metron-platform/metron-parsers-contrib/src/test/resources/dhcp/data_input new file mode 100644 index 0000000000..15f311bc84 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/resources/dhcp/data_input @@ -0,0 +1 @@ +{TIME: 2017-01-16 16:36:17.249|INTERFACE: eth2|OP:1 BOOTPREQUEST|CIADDR: 111.111.111.111|YIADDR: 0.0.0.0|SIADDR: 0.0.0.0|GIADDR: 111.111.111.111|CHADDR: 00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00|OPTION: 53 1 DHCP message type: 8 |DHCPINFORM|OPTION: 61 7 Client-identifier: 00:00:00:00:00:00:00|OPTION: 12 5 Host name: A1234|OPTION: 60 8 Vendor class identifier: MSFT 5.0|OPTION: 55 13 Parameter Request List: 1 (Subnet mask)|| 15 (Domainname)|| 3 (Routers)|| 6 (DNS server)|| 44 (NetBIOS name server)|| 46 (NetBIOS node type)|| 47 (NetBIOS scope)|| 31 (Perform router discovery)|| 33 (Static route)||121 (Classless Static Route)||249 (MSFT - Classless route)|| 43 (Vendor specific info)||252 (MSFT - WinSock Proxy Auto Detect)|||IP: 111.111.111.111 > 111.111.111.111 | 00:00:00:00:00:00 > 00:00:00:00:00:00} \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/dhcp/data_output b/metron-platform/metron-parsers-contrib/src/test/resources/dhcp/data_output new file mode 100644 index 0000000000..55ea64e949 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/resources/dhcp/data_output @@ -0,0 +1 @@ +{"chaddr":"00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00","op":"1 BOOTPREQUEST","mapping":"yyyy-MM-dd HH:mm:ss.SSS","timezone_available":0,"ip":"111.111.111.111 > 111.111.111.111","message_type":"DHCPINFORM","giaddr":"111.111.111.111","interface":"eth2","yiaddr":"0.0.0.0","datetime":"2017-01-16 16:36:17.249","hostname":"A1234","original_string":"{TIME: 2017-01-16 16:36:17.249|INTERFACE: eth2|OP:1 BOOTPREQUEST|CIADDR: 111.111.111.111|YIADDR: 0.0.0.0|SIADDR: 0.0.0.0|GIADDR: 111.111.111.111|CHADDR: 00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00|OPTION: 53 1 DHCP message type: 8 |DHCPINFORM|OPTION: 61 7 Client-identifier: 00:00:00:00:00:00:00|OPTION: 12 5 Host name: A1234|OPTION: 60 8 Vendor class identifier: MSFT 5.0|OPTION: 55 13 Parameter Request List: 1 (Subnet mask)|| 15 (Domainname)|| 3 (Routers)|| 6 (DNS server)|| 44 (NetBIOS name server)|| 46 (NetBIOS node type)|| 47 (NetBIOS scope)|| 31 (Perform router discovery)|| 33 (Static route)||121 (Classless Static Route)||249 (MSFT - Classless route)|| 43 (Vendor specific info)||252 (MSFT - WinSock Proxy Auto Detect)|||IP: 111.111.111.111 > 111.111.111.111 | 00:00:00:00:00:00 > 00:00:00:00:00:00}","original_timestamp":"2017-01-16 16:36:17.249","siaddr":"0.0.0.0","mac_address":"00:00:00:00:00:00:00","time":"2017-01-16 16:36:17.249","ip_src_addr":"111.111.111.111","option":"55 13 Parameter Request List: 1 (Subnet mask)","timestamp":1484584577249} \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/ossec/config.json b/metron-platform/metron-parsers-contrib/src/test/resources/ossec/config.json new file mode 100644 index 0000000000..a1d44d9305 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/resources/ossec/config.json @@ -0,0 +1,43 @@ +{ + "chain": [ + "regex_file_parser", + "parse_timestamp", + "rename_fields" + ], + "parsers": { + "regex_file_parser": { + "class": "org.apache.metron.parsers.contrib.links.io.RegexFileLink", + "patterns": [ + "^AV\\s+\\-\\sAlert\\s+\\-\\s\\\"(?\\d+)\\\"\\s\\-\\->\\sRID\\:\\s\\\"(?\\d+)\\\"\\;\\s+RL\\:\\s+\\\"(?\\d+)\\\"\\;\\s+RG\\:\\s+\\\"(?[^\\\"]*)\\\"\\;\\s+RC\\:\\s+\\\"(?[^\\\"]*)\\\";.*?HOSTNAME\\:\\s*\"?\\((?[^\\)]*)\\)\\s(?\\S+)->.*?AUDIT_SUCCESS\\((?(4634|4647))\\).*?Account\\s+Name\\:\\s+(?.*?)\\s*Account\\s+Domain\\:\\s+(?.*?)\\s*Logon\\s+ID\\:\\s+(?\\S+)(\\s+Logon\\s+Type\\:\\s+(?\\d+))?.*" + ], + "input": "{{original_string}}" + }, + "parse_timestamp": { + "class": "org.apache.metron.parsers.contrib.links.io.TimestampLink", + "patterns": [ + [ + "timestamp", + "newest" + ], + [ + "([0-9]{4})-([0-9]{2})-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2})", + "yyyy MM dd HH mm ss", + "newest" + ], + [ + "([0-9]{4})-([0-9]{2})-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2}) ([+-]{1}[0-9]{1,2}[:][0-9]{2})", + "yyyy MM dd HH mm ss Z", + "newest" + ] + ], + "input": "{{original_string}}" + }, + "rename_fields": { + "class": "org.apache.metron.parsers.contrib.links.fields.RenameLink", + "rename": { + "srcip": "ip_src_addr", + "srcport": "ip_src_port" + } + } + } +} \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/ossec/data_input b/metron-platform/metron-parsers-contrib/src/test/resources/ossec/data_input new file mode 100644 index 0000000000..f425f8c60e --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/resources/ossec/data_input @@ -0,0 +1 @@ +AV - Alert - "1498521603" --> RID: "1234"; RL: "3"; RG: "windows,"; RC: "Windows User Logoff."; USER: "1234"; SRCIP: "None"; HOSTNAME: "(123-nl) 123.123.123.123->WinEvtLog"; LOCATION: "(123-nl) 123.123.123.123->WinEvtLog"; EVENT: "[INIT]2017 Jun 27 01:59:59 WinEvtLog: Security: AUDIT_SUCCESS(4634): Microsoft-Windows-Security-Auditing: 1234: EXID: 123.com: An account was logged off. Subject: Security ID: S-1-5-21-123-123-123-12628 123 Name: 123$ Account Name: myusername Account Domain: EXID Logon ID: 0x222d4cc7 Logon Type: 3 This event is generated when a logon session is destroyed. It may be positively correlated with a logon event using the Logon ID value. Logon IDs are only unique between reboots on the same computer." 4646,1[END]"; \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/ossec/data_output b/metron-platform/metron-parsers-contrib/src/test/resources/ossec/data_output new file mode 100644 index 0000000000..12af202ba7 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/resources/ossec/data_output @@ -0,0 +1 @@ +{"rulecomment":"Windows User Logoff.","mapping":"yyyy-MM-dd HH:mm:ss.SSS Z","timezone_available":1,"logontype":"3","hostname":"123-nl","datetime":"2017-06-27 00:00:03.000 +0000","original_string":"AV - Alert - \"1498521603\" --> RID: \"1234\"; RL: \"3\"; RG: \"windows,\"; RC: \"Windows User Logoff.\"; USER: \"1234\"; SRCIP: \"None\"; HOSTNAME: \"(123-nl) 123.123.123.123->WinEvtLog\"; LOCATION: \"(123-nl) 123.123.123.123->WinEvtLog\"; EVENT: \"[INIT]2017 Jun 27 01:59:59 WinEvtLog: Security: AUDIT_SUCCESS(4634): Microsoft-Windows-Security-Auditing: 1234: EXID: 123.com: An account was logged off. Subject: Security ID: S-1-5-21-123-123-123-12628 123 Name: 123$ Account Name: myusername Account Domain: EXID Logon ID: 0x222d4cc7 Logon Type: 3 This event is generated when a logon session is destroyed. It may be positively correlated with a logon event using the Logon ID value. Logon IDs are only unique between reboots on the same computer.\" 4646,1[END]\";","wineventid":"4634","original_timestamp":"2017-06-27 00:00:03.000","logonid":"0x222d4cc7","rulelevel":"3","domain":"EXID","ruleid":"1234","rulegroup":"windows,","winip":"123.123.123.123","timestamp":1498521603000,"username":"myusername"} \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/suricata/config.json b/metron-platform/metron-parsers-contrib/src/test/resources/suricata/config.json new file mode 100644 index 0000000000..f0897f4497 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/resources/suricata/config.json @@ -0,0 +1,42 @@ +{ + "chain": [ + "parse_json", + "parse_username", + "rename_fields", + "parse_datetime" + ], + "parsers": { + "parse_json": { + "class": "org.apache.metron.parsers.contrib.links.io.JSONDecoderLink" + }, + "parse_username": { + "class": "org.apache.metron.parsers.contrib.links.io.RegexLink", + "pattern": "(?i)(user|username|log)[=:](\\w+)", + "selector": { + "username": "2" + }, + "input": "{{payload_printable}}" + }, + "rename_fields": { + "class": "org.apache.metron.parsers.contrib.links.fields.RenameLink", + "rename": { + "proto": "protocol", + "dest_ip": "ip_dst_addr", + "src_ip": "ip_src_addr", + "dest_port": "ip_dst_port", + "src_port": "ip_src_port" + } + }, + "parse_datetime": { + "class": "org.apache.metron.parsers.contrib.links.io.TimestampLink", + "patterns": [ + [ + "([0-9]{4})-([0-9]+)-([0-9]+)T([0-9]+):([0-9]+):([0-9]+).([0-9]+)([+-]{1}[0-9]{1,2}[:]?[0-9]{2})", + "yyyy MM dd HH mm ss SSSSSS Z", + "newest" + ] + ], + "input": "{{timestamp}}" + } + } +} \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/suricata/data_input b/metron-platform/metron-parsers-contrib/src/test/resources/suricata/data_input new file mode 100644 index 0000000000..731679f19e --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/resources/suricata/data_input @@ -0,0 +1,3 @@ +{"timestamp":"2017-01-11T00:54:01.918498+0100","flow_id":111,"in_iface":"eth0","event_type":"dns","src_ip":"111.111.111.111","src_port":1234,"dest_ip":"111.111.111.111","dest_port":1234,"proto":"UDP","dns":{"type":"query","id":1111,"rrname":"x.local","rrtype":"A","tx_id":0}} +{"timestamp":"2017-09-07T15:10:30.292276+0200","flow_id":111,"in_iface":"eth0","event_type":"dns","vlan":1,"src_ip":"111.111.111.111","src_port":1234,"dest_ip":"111.111.111.111","dest_port":1234,"proto":"UDP","dns":{"type":"answer","id":111,"rcode":"NOERROR","rrname":"x.local","rrtype":"NS","ttl":3600,"rdata":"y.local"}} +{"timestamp":"2017-10-23T14:15:58.514793+0200","flow_id":111,"in_iface":"eth0","event_type":"alert","vlan":1,"src_ip":"111.111.111.111","src_port":1234,"dest_ip":"111.111.111.111","dest_port":80,"proto":"TCP","tx_id":0,"alert":{"action":"allowed","gid":1,"signature_id":1234,"rev":3,"signature":"ET POLICY Http Client Body contains pwd= in cleartext","category":"Potential Corporate Privacy Violation","severity":1},"http":{"hostname":"x.com","url":"\/login.php?op=cusLogin","http_user_agent":"Apache-HttpClient\/UNAVAILABLE (java 1.4)","http_content_type":"application\/json","http_method":"POST","protocol":"HTTP\/1.1","status":200,"length":60},"payload":"nopayload","payload_printable":"pwd=1234&userName=user1","stream":1,"packet":"1234","packet_info":{"linktype":1}} \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/suricata/data_output b/metron-platform/metron-parsers-contrib/src/test/resources/suricata/data_output new file mode 100644 index 0000000000..9bb69dbc84 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/resources/suricata/data_output @@ -0,0 +1,3 @@ +{"dns.type":"query","mapping":"yyyy-MM-dd HH:mm:ss.SSS Z","ip_dst_port":1234,"timezone_available":1,"dns.id":1111,"in_iface":"eth0","dns.rrname":"x.local","protocol":"UDP","datetime":"2017-01-11 00:54:01.918 +0100","original_string":"{\"timestamp\":\"2017-01-11T00:54:01.918498+0100\",\"flow_id\":111,\"in_iface\":\"eth0\",\"event_type\":\"dns\",\"src_ip\":\"111.111.111.111\",\"src_port\":1234,\"dest_ip\":\"111.111.111.111\",\"dest_port\":1234,\"proto\":\"UDP\",\"dns\":{\"type\":\"query\",\"id\":1111,\"rrname\":\"x.local\",\"rrtype\":\"A\",\"tx_id\":0}}","event_type":"dns","dns.tx_id":0,"ip_dst_addr":"111.111.111.111","original_timestamp":"2017-01-11 00:54:01.918","ip_src_port":1234,"flow_id":111,"dns.rrtype":"A","ip_src_addr":"111.111.111.111","timestamp":1484092441918} +{"dns.type":"answer","mapping":"yyyy-MM-dd HH:mm:ss.SSS Z","ip_dst_port":1234,"timezone_available":1,"dns.rcode":"NOERROR","dns.id":111,"dns.ttl":3600,"dns.rdata":"y.local","in_iface":"eth0","dns.rrname":"x.local","protocol":"UDP","datetime":"2017-09-07 15:10:30.292 +0200","original_string":"{\"timestamp\":\"2017-09-07T15:10:30.292276+0200\",\"flow_id\":111,\"in_iface\":\"eth0\",\"event_type\":\"dns\",\"vlan\":1,\"src_ip\":\"111.111.111.111\",\"src_port\":1234,\"dest_ip\":\"111.111.111.111\",\"dest_port\":1234,\"proto\":\"UDP\",\"dns\":{\"type\":\"answer\",\"id\":111,\"rcode\":\"NOERROR\",\"rrname\":\"x.local\",\"rrtype\":\"NS\",\"ttl\":3600,\"rdata\":\"y.local\"}}","event_type":"dns","ip_dst_addr":"111.111.111.111","original_timestamp":"2017-09-07 15:10:30.292","vlan":1,"ip_src_port":1234,"flow_id":111,"dns.rrtype":"NS","ip_src_addr":"111.111.111.111","timestamp":1504789830292} +{"alert.category":"Potential Corporate Privacy Violation","ip_dst_port":80,"http.http_method":"POST","timezone_available":1,"http.url":"\/login.php?op=cusLogin","http.http_content_type":"application\/json","protocol":"TCP","datetime":"2017-10-23 14:15:58.514 +0200","original_string":"{\"timestamp\":\"2017-10-23T14:15:58.514793+0200\",\"flow_id\":111,\"in_iface\":\"eth0\",\"event_type\":\"alert\",\"vlan\":1,\"src_ip\":\"111.111.111.111\",\"src_port\":1234,\"dest_ip\":\"111.111.111.111\",\"dest_port\":80,\"proto\":\"TCP\",\"tx_id\":0,\"alert\":{\"action\":\"allowed\",\"gid\":1,\"signature_id\":1234,\"rev\":3,\"signature\":\"ET POLICY Http Client Body contains pwd= in cleartext\",\"category\":\"Potential Corporate Privacy Violation\",\"severity\":1},\"http\":{\"hostname\":\"x.com\",\"url\":\"\\\/login.php?op=cusLogin\",\"http_user_agent\":\"Apache-HttpClient\\\/UNAVAILABLE (java 1.4)\",\"http_content_type\":\"application\\\/json\",\"http_method\":\"POST\",\"protocol\":\"HTTP\\\/1.1\",\"status\":200,\"length\":60},\"payload\":\"nopayload\",\"payload_printable\":\"pwd=1234&userName=user1\",\"stream\":1,\"packet\":\"1234\",\"packet_info\":{\"linktype\":1}}","event_type":"alert","ip_dst_addr":"111.111.111.111","vlan":1,"alert.rev":3,"payload":"nopayload","stream":1,"flow_id":111,"alert.signature_id":1234,"alert.action":"allowed","packet_info.linktype":1,"ip_src_addr":"111.111.111.111","timestamp":1508760958514,"alert.severity":1,"mapping":"yyyy-MM-dd HH:mm:ss.SSS Z","payload_printable":"pwd=1234&userName=user1","tx_id":0,"packet":"1234","http.protocol":"HTTP\/1.1","http.length":60,"in_iface":"eth0","http.hostname":"x.com","alert.gid":1,"original_timestamp":"2017-10-23 14:15:58.514","alert.signature":"ET POLICY Http Client Body contains pwd= in cleartext","ip_src_port":1234,"http.status":200,"http.http_user_agent":"Apache-HttpClient\/UNAVAILABLE (java 1.4)","username":"user1"} \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/yaf/config.json b/metron-platform/metron-parsers-contrib/src/test/resources/yaf/config.json new file mode 100644 index 0000000000..831387a988 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/resources/yaf/config.json @@ -0,0 +1,69 @@ +{ + "chain": [ + "values_splitter", + "trim_whitespaces", + "parse_datetime", + "rename_fields" + ], + "parsers": { + "values_splitter": { + "class": "org.apache.metron.parsers.contrib.links.io.SplitLink", + "delimiter": "|", + "selector": { + "1": "start-time", + "2": "end-time", + "3": "duration", + "4": "rtt", + "5": "proto", + "6": "sip", + "7": "sp", + "8": "dip", + "9": "dp", + "10": "srcMacAddress", + "11": "destMacAddress", + "12": "iflags", + "13": "uflags", + "14": "riflags", + "15": "ruflags", + "16": "isn", + "17": "risn", + "18": "tag", + "19": "rtag", + "20": "pkt", + "21": "oct", + "22": "rpkt", + "23": "roct", + "24": "app", + "25": "end-reason" + } + }, + "parse_datetime": { + "class": "org.apache.metron.parsers.contrib.links.io.TimestampLink", + "patterns": [ + [ + "([0-9]{4})-([0-9]{2})-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2}).([0-9]{3})", + "yyyy MM dd HH mm ss SSS", + "newest" + ], + [ + "([0-9]{4})-([0-9]{2})-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2}).([0-9]{3}) ([+-]{1}[0-9]{1,2}[:][0-9]{2})", + "yyyy MM dd HH mm ss SSS Z", + "newest" + ] + ], + "input": "{{end-time}}" + }, + "trim_whitespaces": { + "class" : "org.apache.metron.parsers.contrib.links.fields.TrimValueLink" + }, + "rename_fields": { + "class": "org.apache.metron.parsers.contrib.links.fields.RenameLink", + "rename": { + "sip": "ip_src_addr", + "sp": "ip_src_port", + "dip": "ip_dst_addr", + "dp": "ip_dst_port" + } + } + } +} \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/yaf/data_input b/metron-platform/metron-parsers-contrib/src/test/resources/yaf/data_input new file mode 100644 index 0000000000..62a198eee0 --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/resources/yaf/data_input @@ -0,0 +1 @@ +{2016-12-14 09:36:31.181|2016-12-14 09:36:31.442| 0.261| 0.000| 6| 111.111.111.111|11210| 111.111.111.111| 443|00:00:00:00:00:00|00:00:00:00:00:00| S| APF| AS| APF|aaa|aaa|000|000| 19| 4811| 16| 7059| 0|} \ No newline at end of file diff --git a/metron-platform/metron-parsers-contrib/src/test/resources/yaf/data_output b/metron-platform/metron-parsers-contrib/src/test/resources/yaf/data_output new file mode 100644 index 0000000000..daba543e7a --- /dev/null +++ b/metron-platform/metron-parsers-contrib/src/test/resources/yaf/data_output @@ -0,0 +1 @@ +{"iflags":"APF","uflags":"AS","ip_dst_port":"00:00:00:00:00:00","isn":"aaa","timezone_available":0,"destMacAddress":"S","duration":"0.000","rpkt":"7059","datetime":"2016-12-14 09:36:31.442","original_string":"{2016-12-14 09:36:31.181|2016-12-14 09:36:31.442| 0.261| 0.000| 6| 111.111.111.111|11210| 111.111.111.111| 443|00:00:00:00:00:00|00:00:00:00:00:00| S| APF| AS| APF|aaa|aaa|000|000| 19| 4811| 16| 7059| 0|}","ip_dst_addr":"443","pkt":"4811","ruflags":"aaa","tag":"000","roct":"0","rtag":"19","ip_src_addr":"11210","timestamp":1481708191442,"app":"}","oct":"16","mapping":"yyyy-MM-dd HH:mm:ss.SSS","start-time":"2016-12-14 09:36:31.442","end-time":"0.261","risn":"000","rtt":"6","riflags":"APF","original_timestamp":"2016-12-14 09:36:31.442","ip_src_port":"111.111.111.111","proto":"111.111.111.111","srcMacAddress":"00:00:00:00:00:00"} \ No newline at end of file diff --git a/metron-platform/pom.xml b/metron-platform/pom.xml index 3899e9def7..708a526464 100644 --- a/metron-platform/pom.xml +++ b/metron-platform/pom.xml @@ -46,6 +46,7 @@ metron-enrichment metron-solr metron-parsers + metron-parsers-contrib metron-pcap-backend metron-data-management metron-pcap diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarBaseListener.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarBaseListener.java index 3528737133..9915837fc1 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarBaseListener.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarBaseListener.java @@ -1,4 +1,4 @@ -// Generated from org/apache/metron/stellar/common/generated/Stellar.g4 by ANTLR 4.5 +// Generated from org\apache\metron\stellar\common\generated\Stellar.g4 by ANTLR 4.5 package org.apache.metron.stellar.common.generated; //CHECKSTYLE:OFF diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarLexer.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarLexer.java index df661a9159..267d5936eb 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarLexer.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarLexer.java @@ -1,4 +1,4 @@ -// Generated from org/apache/metron/stellar/common/generated/Stellar.g4 by ANTLR 4.5 +// Generated from org\apache\metron\stellar\common\generated\Stellar.g4 by ANTLR 4.5 package org.apache.metron.stellar.common.generated; //CHECKSTYLE:OFF diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarListener.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarListener.java index 718a4fe565..66d273a196 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarListener.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarListener.java @@ -1,4 +1,4 @@ -// Generated from org/apache/metron/stellar/common/generated/Stellar.g4 by ANTLR 4.5 +// Generated from org\apache\metron\stellar\common\generated\Stellar.g4 by ANTLR 4.5 package org.apache.metron.stellar.common.generated; //CHECKSTYLE:OFF diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarParser.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarParser.java index 3bd4ad161f..21c54153d0 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarParser.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarParser.java @@ -1,4 +1,4 @@ -// Generated from org/apache/metron/stellar/common/generated/Stellar.g4 by ANTLR 4.5 +// Generated from org\apache\metron\stellar\common\generated\Stellar.g4 by ANTLR 4.5 package org.apache.metron.stellar.common.generated; //CHECKSTYLE:OFF