remove most of was pointers - fixed a conflict between AWS plugin and…

… NooBaa

problems before this fix:

1. data input of AWS appeared in NooBaa
2. NooBaa data input failed

Still open issue - click on noobaa icon (launcher) failed with 404 (as
it redirects to /inputs)

Splunk/Splunk_NooBaa/README/inputs.conf.spec

@@ -1,23 +1,5 @@
-[aws_cloudtrail://<name>]
-aws_account = AWS account used to connect to AWS
-aws_region = AWS region of log notification SQS queue
-sqs_queue = log notification SQS queue
-exclude_describe_events = boolean indicating whether to exclude read-only events from indexing. defaults to true
-remove_files_when_done = boolean indicating whether to remove s3 files after reading defaults to false
-blacklist = override regex for the "exclude_describe_events" setting. default regex applied is ^(?:Describe|List|Get)
-excluded_events_index = name of index to put excluded events into. default is empty, which discards the events
-
-[aws_cloudwatch://<name>]
-aws_account = AWS account used to connect to AWS
-aws_region = AWS region of CloudWatch metrics
-metric_namespace = CloudWatch metric namespace
-metric_names = CloudWatch metric names
-metric_dimensions = CloudWatch metric dimensions
-statistics = CloudWatch metric statistics being requested
-period = CloudWatch metric granularity
-polling_interval = Polling interval for statistics
 
-[aws_s3://<name>]
+[noobaa_s3://<name>]
 is_secure = whether use secure connection to AWS
 host_name = the host name of the S3 service
 aws_account = AWS account used to connect to AWS
@@ -30,30 +12,3 @@ max_retries = Max number of retry attempts to stream incomplete items.
 whitelist = Override regex for blacklist when using a folder key.
 blacklist = Keys to ignore when using a folder key.
 character_set = The encoding used in your S3 files. Default to 'auto' meaning that file encoding will be detected automatically amoung UTF-8, UTF8 without BOM, UTF-16BE, UTF-16LE, UTF32BE and UTF32LE. Notice that once one specified encoding is set, data input will only handle that encoding.
-
-[aws_billing://<name>]
-aws_account = AWS account used to connect to fetch the billing report
-host_name = the host name of the S3 service
-bucket_name = S3 bucket
-report_file_match_reg = CSV report file in regex, it will override below report type options instead
-monthly_report_type = report type for monthly report. options: None, Monthly report, Monthly cost allocation report
-detail_report_type = report type for detail report. options: None, Detailed billing report, Detailed billing report with resources and tags
-
-# below items are internally used only
-recursion_depth = recursion depth when iterate files
-initial_scan_datetime = this option is deprecated
-monthly_timestamp_select_column_list = fields of timestamp extracted from monthly report, seperated by '|'
-detail_timestamp_select_column_list = fields of timestamp extracted from detail report, seperated by '|'
-time_format_list = time format extraction from existing. e.g. "%Y-%m-%d %H:%M:%S" seperated by '|'
-max_file_size_csv_in_bytes = max file size in csv file format, default: 50MB
-max_file_size_csv_zip_in_bytes = max file size in csv zip format, default: 1GB
-header_look_up_max_lines = maximum lines to look up header of billing report
-header_magic_regex = regex of header to look up
-monthly_real_timestamp_extraction = for monthly report, regex to extract real timestamp in the montlh report, must contains "(%TIME_FORMAT_REGEX%)", which will be replaced with one value defined in "monthly_real_timestamp_format_reg_list"
-monthly_real_timestamp_format_reg_list = for monthly report, regex to match the format of real time string. seperated by '|'
-
-[aws_config://<name>]
-aws_account = AWS account used to connect to AWS
-aws_region = AWS region of log notification SQS queue
-sqs_queue = Starling Notification SQS queue
-enable_additional_notifications = Enable collection of additional helper notifications

Splunk/Splunk_NooBaa/bin/aws_s3.py → Splunk/Splunk_NooBaa/bin/noobaa_s3.py

@@ -179,8 +179,8 @@ def __init__(self, checkpoint_dir, stanza_name, bucket_name, server_uri,
         self._max_items = max_items
 
         start_offset = 0
-        if stanza_name.startswith('aws-s3://'):
-            start_offset = len('aws-s3://')
+        if stanza_name.startswith('noobaa-s3://'):
+            start_offset = len('noobaa-s3://')
 
         safe_filename_prefix = "".join([c if c.isalnum() else '_' for c in stanza_name[start_offset:start_offset + 20]])
         stanza_hexdigest = hashlib.md5("{}_{}".format(stanza_name, bucket_name)).hexdigest()

Splunk/Splunk_NooBaa/bin/splunk_ta_aws_log_level_handler.py → Splunk/Splunk_NooBaa/bin/splunk_noobaa_log_level_handler.py

@@ -26,12 +26,7 @@ def wrapper(*args, **kwargs):
         return result
     return wrapper
 
-DATA_INPUTS=['aws_billing',
-      'aws_cloudtrail',
-      'aws_cloudwatch',
-      'aws_config',
-      'aws_s3',
-      ]
+DATA_INPUTS=['noobaa_s3']
 
 class ConfigHandler(splunk.admin.MConfigHandler):
 

Splunk/Splunk_NooBaa/bin/splunk_ta_aws_s3key_handler.py → Splunk/Splunk_NooBaa/bin/splunk_noobaa_s3key_handler.py

@@ -18,7 +18,7 @@
 from taaws.log import setup_logger
 from taaws.aws_accesskeys import AwsAccessKeyManager
 from taaws.s3util import connect_s3
-from splunk_ta_aws_s3buckets_handler import timed
+from splunk_noobaa_s3buckets_handler import timed
 logger = setup_logger(APPNAME + '-RestEndpoints', level=logging.DEBUG)
 
 

Splunk/Splunk_NooBaa/default/app.conf

@@ -19,7 +19,7 @@ docs_section_override = AddOns:released
 
 [launcher]
 author = Splunk
-version = 3.0
+version = 3.1.4
 description = Splunk Add-on for NooBaa
 
 [package]

Splunk/Splunk_NooBaa/default/data/ui/manager/data_inputs_aws-billing.xml

@@ -29,7 +29,7 @@
           <key name="dynamicOptions" type="dict">
             <key name="keyName">entry</key>
             <key name="entityField">'accounts'</key>
-            <key name="splunkSource">'/admin/splunk_ta_aws_accout_list/'</key>
+            <key name="splunkSource">'/admin/splunk_noobaa_account_list/'</key>
             <key name="splunkSourceParams" type="dict">
                 <key name="aws_account_service">'all'</key>
             </key>
@@ -65,7 +65,7 @@
           <key name="dynamicOptions" type="dict">
             <key name="keyName">entry</key>
             <key name="entityField">'buckets'</key>
-            <key name="splunkSource">'/admin/splunk_ta_aws_s3buckets/'</key>
+            <key name="splunkSource">'/admin/splunk_noobaa_s3buckets/'</key>
             <key name="splunkSourceParams" type="dict">
             	<key name="aws_account">form_defaults['aws_account']</key>
                 <key name="host_name">form_defaults['host_name']</key>
@@ -105,7 +105,7 @@
           <key name="dynamicOptions" type="dict">
             <key name="keyName">entry</key>
             <key name="entityField">'monthly_report'</key>
-            <key name="splunkSource">'/admin/splunk_ta_aws_billing_report_types/'</key>
+            <key name="splunkSource">'/admin/splunk_noobaa_billing_report_types/'</key>
             <key name="splunkSourceParams" type="dict">
             </key>
             <key name="splunkSourceEntity">'ReportTypesResultList'</key>
@@ -120,7 +120,7 @@
           <key name="dynamicOptions" type="dict">
             <key name="keyName">entry</key>
             <key name="entityField">'detail_report'</key>
-            <key name="splunkSource">'/admin/splunk_ta_aws_billing_report_types/'</key>
+            <key name="splunkSource">'/admin/splunk_noobaa_billing_report_types/'</key>
             <key name="splunkSourceParams" type="dict">
             </key>
             <key name="splunkSourceEntity">'ReportTypesResultList'</key>
@@ -286,4 +286,4 @@
 
 
   </elements>
-</endpoint>
+</endpoint>

Splunk/Splunk_NooBaa/default/data/ui/manager/data_inputs_aws-cloudtrail.xml

@@ -27,7 +27,7 @@
           <key name="dynamicOptions" type="dict">
             <key name="keyName">entry</key>
             <key name="entityField">'accounts'</key>
-            <key name="splunkSource">'/admin/splunk_ta_aws_accout_list/'</key>
+            <key name="splunkSource">'/admin/splunk_noobaa_account_list/'</key>
             <key name="splunkSourceParams" type="dict">
                 <key name="aws_account_service">'all'</key>
             </key>
@@ -55,7 +55,7 @@
 		  <key name="dynamicOptions" type="dict">
 			<key name="keyName">entry</key>
 			<key name="entityField">'regions'</key>
-			<key name="splunkSource">'/admin/splunk_ta_aws_regions/'</key>
+			<key name="splunkSource">'/admin/splunk_noobaa_regions/'</key>
 			<key name="splunkSourceParams" type="dict">
 			  <key name="aws_service">'cloudtrail'</key>
 			</key>
@@ -95,7 +95,7 @@
 			  <key name="dynamicOptions" type="dict">
 				<key name="keyName">entry</key>
 				<key name="entityField">'sqs_queue_names'</key>
-				<key name="splunkSource">'/admin/splunk_ta_aws_sqs_queue_names/'</key>
+				<key name="splunkSource">'/admin/splunk_noobaa_sqs_queue_names/'</key>
 				<key name="splunkSourceParams" type="dict">
 				  <key name="aws_region">form_defaults['aws_region']</key>
 				  <key name="aws_account">form_defaults['aws_account']</key>

Splunk/Splunk_NooBaa/default/data/ui/manager/data_inputs_aws-cloudwatch.xml

@@ -27,7 +27,7 @@
           <key name="dynamicOptions" type="dict">
             <key name="keyName">entry</key>
             <key name="entityField">'accounts'</key>
-            <key name="splunkSource">'/admin/splunk_ta_aws_accout_list/'</key>
+            <key name="splunkSource">'/admin/splunk_noobaa_account_list/'</key>
             <key name="splunkSourceParams" type="dict">
                 <key name="aws_account_service">'all'</key>
             </key>
@@ -56,7 +56,7 @@
           <key name="dynamicOptions" type="dict">
           <key name="keyName">entry</key>
           <key name="entityField">'regions'</key>
-          <key name="splunkSource">'/admin/splunk_ta_aws_regions/'</key>
+          <key name="splunkSource">'/admin/splunk_noobaa_regions/'</key>
           <key name="splunkSourceParams" type="dict">
             <key name="aws_service">'cloudtrail'</key>
           </key>
@@ -85,7 +85,7 @@
           <key name="dynamicOptions" type="dict">
           <key name="keyName">entry</key>
           <key name="entityField">'metric_namespace'</key>
-          <key name="splunkSource">'/admin/splunk_ta_aws_namespaces/'</key>
+          <key name="splunkSource">'/admin/splunk_noobaa_namespaces/'</key>
           <key name="splunkSourceParams" type="dict">
             <key name="aws_region">form_defaults['aws_region']</key>
             <key name="aws_account">form_defaults['aws_account']</key>

Splunk/Splunk_NooBaa/default/data/ui/manager/data_inputs_aws-config.xml

@@ -27,7 +27,7 @@
           <key name="dynamicOptions" type="dict">
             <key name="keyName">entry</key>
             <key name="entityField">'accounts'</key>
-            <key name="splunkSource">'/admin/splunk_ta_aws_accout_list/'</key>
+            <key name="splunkSource">'/admin/splunk_noobaa_account_list/'</key>
             <key name="splunkSourceParams" type="dict">
                 <key name="aws_account_service">'all'</key>
             </key>
@@ -55,7 +55,7 @@
 		  <key name="dynamicOptions" type="dict">
 			<key name="keyName">entry</key>
 			<key name="entityField">'regions'</key>
-			<key name="splunkSource">'/admin/splunk_ta_aws_regions/'</key>
+			<key name="splunkSource">'/admin/splunk_noobaa_regions/'</key>
 			<key name="splunkSourceParams" type="dict">
 			  <key name="aws_service">'awsconfig'</key>
 			</key>
@@ -91,7 +91,7 @@
 			  <key name="dynamicOptions" type="dict">
 				<key name="keyName">entry</key>
 				<key name="entityField">'sqs_queue_names'</key>
-				<key name="splunkSource">'/admin/splunk_ta_aws_sqs_queue_names/'</key>
+				<key name="splunkSource">'/admin/splunk_noobaa_sqs_queue_names/'</key>
 				<key name="splunkSourceParams" type="dict">
 				  <key name="aws_region">form_defaults['aws_region']</key>
 				  <key name="aws_account">form_defaults['aws_account']</key>

Splunk/Splunk_NooBaa/default/data/ui/manager/data_inputs_aws-s3.xml → Splunk/Splunk_NooBaa/default/data/ui/manager/data_inputs_noobaa-s3.xml

@@ -1,16 +1,16 @@
-<endpoint name="data/inputs/aws_s3">
+<endpoint name="data/inputs/noobaa_s3">
   <header>NOOBAA S3 API</header>
   <breadcrumb>
     <parent hidecurrent="False">datainputstats</parent>
     <name i18ntag="">NOOBAA S3 API</name>
   </breadcrumb>
   <elements>
-    
+
     <element name="name" label="Input Name">
       <view name="list"/>
       <view name="create"/>
     </element>
-    
+
     <element name="sourceFields" type="fieldset">
       <key name="legend">NooBaa S3 REST gateway Configuration</key>
       <view name="list"/>
@@ -28,7 +28,7 @@
           <key name="dynamicOptions" type="dict">
             <key name="keyName">entry</key>
             <key name="entityField">'accounts'</key>
-            <key name="splunkSource">'/admin/splunk_ta_aws_accout_list/'</key>
+            <key name="splunkSource">'/admin/splunk_noobaa_account_list/'</key>
             <key name="splunkSourceParams" type="dict">
                 <key name="aws_account_service">'all'</key>
             </key>
@@ -127,7 +127,7 @@
           <view name="edit"/>
           <view name="create"/>
         </element>
-        
+
         <element name="character_set" type="textfield" label="Character set">
           <key name="exampleText">
               Specify the encoding used in your S3 files.
@@ -156,13 +156,13 @@
               </group_set>
           </onChange>
     </element>
-    
+
     <element name="advanced" type="fieldset" class="spl-mgr-advanced-options">
       <view name="list"/>
       <view name="edit"/>
       <view name="create"/>
       <elements>
-        
+
         <element name="intervalFields" type="fieldset">
           <key name="legend">Interval</key>
           <view name="edit"/>
@@ -177,7 +177,7 @@
                   <view name="create"/>
                 </element>
               </elements>
-        </element> 
+        </element>
 
         <element name="sourcetypeFields" type="fieldset">
           <view name="list"/>
@@ -256,7 +256,7 @@
               <view name="create"/>
             </element>
           </elements>
-        </element>  
+        </element>
         <element name="indexField" type="fieldset">
           <key name="legend">Index</key>
           <key name="helpText">Set the destination index for this source.</key>

Splunk/Splunk_NooBaa/default/inputs.conf

@@ -1,23 +1,7 @@
 
-[aws_cloudtrail]
+[noobaa_s3]
 aws_account =
-sourcetype = aws:cloudtrail
-exclude_describe_events = true
-remove_files_when_done = false
-queueSize = 128KB
-persistentQueueSize = 24MB
-interval = 30
-
-[aws_cloudwatch]
-aws_account =
-sourcetype = aws:cloudwatch
-queueSize = 128KB
-persistentQueueSize = 24MB
-interval = 30
-
-[aws_s3]
-aws_account =
-sourcetype = aws:s3
+sourcetype = noobaa:s3
 initial_scan_datetime = default
 max_items = 100000
 max_retries = 10
@@ -27,39 +11,4 @@ interval = 1800
 recursion_depth = -1
 character_set = auto
 is_secure = True
-host_name = s3.amazonaws.com
-
-
-[aws_billing]
-bucket_name =
-aws_account =
-monthly_report_type = Monthly cost allocation report
-detail_report_type = Detailed billing report with resources and tags
-report_file_match_reg =
-interval = 86400
-sourcetype = aws:billing
-host_name = s3.amazonaws.com
-
-# below settings are used internally only
-recursion_depth = 0
-initial_scan_datetime = 0
-time_format_list = %m/%d/%y %H:%M | %Y-%m-%d %H:%M:%S | %Y/%m/%d %H:%M:%S
-monthly_timestamp_select_column_list = BillingPeriodEndDate
-detail_timestamp_select_column_list = UsageStartDate
-max_file_size_csv_in_bytes = 52428800
-max_file_size_csv_zip_in_bytes = 1048576000
-header_look_up_max_lines = 3
-header_magic_regex = InvoiceID[,\w\s]*PayerAccountId
-monthly_real_timestamp_extraction = ["]?EstimatedDisclaimer["]?.+["]?This report.+through approximately (%TIME_FORMAT_REGEX%)["]?,
-monthly_real_timestamp_format_reg_list = .+?
-
-[aws_config]
-aws_account = 
-sourcetype = aws:config
-#exclude_describe_events = true
-enable_additional_notifications = false
-queueSize = 128KB
-persistentQueueSize = 24MB
-interval = 30
-
-
+host_name =

Splunk/Splunk_NooBaa/default/log_info.conf

@@ -1,15 +1,2 @@
-
-[aws_billing]
-level=INFO
-
-[aws_cloudtrail]
-level=INFO
-
-[aws_cloudwatch]
-level=INFO
-
-[aws_config]
-level=INFO
-
-[aws_s3]
+[noobaa_s3]
 level=INFO