forked from IBMStreams/samples
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
HDFSFormat sample plus some gitignore changes
- Loading branch information
Showing
25 changed files
with
1,142 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| toolkit.xml | ||
| output | ||
| doc |
3 changes: 3 additions & 0 deletions
3
HDFSFormatter/hdfs2.helpers.samples/HDFSJSONFormat/.gitignore
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| /output | ||
| data/jsonReadTest.csv | ||
| /doc |
29 changes: 29 additions & 0 deletions
29
HDFSFormatter/hdfs2.helpers.samples/HDFSJSONFormat/.project
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| <?xml version="1.0" encoding="UTF-8"?> | ||
| <projectDescription> | ||
| <name>HDFSJSONFormat</name> | ||
| <comment></comment> | ||
| <projects> | ||
| </projects> | ||
| <buildSpec> | ||
| <buildCommand> | ||
| <name>org.eclipse.jdt.core.javabuilder</name> | ||
| <arguments> | ||
| </arguments> | ||
| </buildCommand> | ||
| <buildCommand> | ||
| <name>com.ibm.streams.studio.splproject.builder.SPLProjectBuilder</name> | ||
| <arguments> | ||
| </arguments> | ||
| </buildCommand> | ||
| <buildCommand> | ||
| <name>org.eclipse.xtext.ui.shared.xtextBuilder</name> | ||
| <arguments> | ||
| </arguments> | ||
| </buildCommand> | ||
| </buildSpec> | ||
| <natures> | ||
| <nature>com.ibm.streams.studio.splproject.SPLProjectNature</nature> | ||
| <nature>org.eclipse.xtext.ui.shared.xtextNature</nature> | ||
| <nature>org.eclipse.jdt.core.javanature</nature> | ||
| </natures> | ||
| </projectDescription> |
11 changes: 11 additions & 0 deletions
11
HDFSFormatter/hdfs2.helpers.samples/HDFSJSONFormat/LICENSE.md
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| ##License | ||
|
|
||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this except in compliance with the License. | ||
|
|
||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
|
|
||
| Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
|
|
||
| See the License for the specific language governing permissions and limitations under the License. |
100 changes: 100 additions & 0 deletions
100
HDFSFormatter/hdfs2.helpers.samples/HDFSJSONFormat/data/inputForJSONRead.txt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| {"aString":"Number is 0","n":1} | ||
| {"aString":"Number is 2","n":3} | ||
| {"aString":"Number is 4","n":5} | ||
| {"aString":"Number is 6","n":7} | ||
| {"aString":"Number is 8","n":9} | ||
| {"aString":"Number is 10","n":11} | ||
| {"aString":"Number is 12","n":13} | ||
| {"aString":"Number is 14","n":15} | ||
| {"aString":"Number is 16","n":17} | ||
| {"aString":"Number is 18","n":19} | ||
| {"aString":"Number is 20","n":21} | ||
| {"aString":"Number is 22","n":23} | ||
| {"aString":"Number is 24","n":25} | ||
| {"aString":"Number is 26","n":27} | ||
| {"aString":"Number is 28","n":29} | ||
| {"aString":"Number is 30","n":31} | ||
| {"aString":"Number is 32","n":33} | ||
| {"aString":"Number is 34","n":35} | ||
| {"aString":"Number is 36","n":37} | ||
| {"aString":"Number is 38","n":39} | ||
| {"aString":"Number is 40","n":41} | ||
| {"aString":"Number is 42","n":43} | ||
| {"aString":"Number is 44","n":45} | ||
| {"aString":"Number is 46","n":47} | ||
| {"aString":"Number is 48","n":49} | ||
| {"aString":"Number is 50","n":51} | ||
| {"aString":"Number is 52","n":53} | ||
| {"aString":"Number is 54","n":55} | ||
| {"aString":"Number is 56","n":57} | ||
| {"aString":"Number is 58","n":59} | ||
| {"aString":"Number is 60","n":61} | ||
| {"aString":"Number is 62","n":63} | ||
| {"aString":"Number is 64","n":65} | ||
| {"aString":"Number is 66","n":67} | ||
| {"aString":"Number is 68","n":69} | ||
| {"aString":"Number is 70","n":71} | ||
| {"aString":"Number is 72","n":73} | ||
| {"aString":"Number is 74","n":75} | ||
| {"aString":"Number is 76","n":77} | ||
| {"aString":"Number is 78","n":79} | ||
| {"aString":"Number is 80","n":81} | ||
| {"aString":"Number is 82","n":83} | ||
| {"aString":"Number is 84","n":85} | ||
| {"aString":"Number is 86","n":87} | ||
| {"aString":"Number is 88","n":89} | ||
| {"aString":"Number is 90","n":91} | ||
| {"aString":"Number is 92","n":93} | ||
| {"aString":"Number is 94","n":95} | ||
| {"aString":"Number is 96","n":97} | ||
| {"aString":"Number is 98","n":99} | ||
| {"aString":"Number is 100","n":101} | ||
| {"aString":"Number is 102","n":103} | ||
| {"aString":"Number is 104","n":105} | ||
| {"aString":"Number is 106","n":107} | ||
| {"aString":"Number is 108","n":109} | ||
| {"aString":"Number is 110","n":111} | ||
| {"aString":"Number is 112","n":113} | ||
| {"aString":"Number is 114","n":115} | ||
| {"aString":"Number is 116","n":117} | ||
| {"aString":"Number is 118","n":119} | ||
| {"aString":"Number is 120","n":121} | ||
| {"aString":"Number is 122","n":123} | ||
| {"aString":"Number is 124","n":125} | ||
| {"aString":"Number is 126","n":127} | ||
| {"aString":"Number is 128","n":129} | ||
| {"aString":"Number is 130","n":131} | ||
| {"aString":"Number is 132","n":133} | ||
| {"aString":"Number is 134","n":135} | ||
| {"aString":"Number is 136","n":137} | ||
| {"aString":"Number is 138","n":139} | ||
| {"aString":"Number is 140","n":141} | ||
| {"aString":"Number is 142","n":143} | ||
| {"aString":"Number is 144","n":145} | ||
| {"aString":"Number is 146","n":147} | ||
| {"aString":"Number is 148","n":149} | ||
| {"aString":"Number is 150","n":151} | ||
| {"aString":"Number is 152","n":153} | ||
| {"aString":"Number is 154","n":155} | ||
| {"aString":"Number is 156","n":157} | ||
| {"aString":"Number is 158","n":159} | ||
| {"aString":"Number is 160","n":161} | ||
| {"aString":"Number is 162","n":163} | ||
| {"aString":"Number is 164","n":165} | ||
| {"aString":"Number is 166","n":167} | ||
| {"aString":"Number is 168","n":169} | ||
| {"aString":"Number is 170","n":171} | ||
| {"aString":"Number is 172","n":173} | ||
| {"aString":"Number is 174","n":175} | ||
| {"aString":"Number is 176","n":177} | ||
| {"aString":"Number is 178","n":179} | ||
| {"aString":"Number is 180","n":181} | ||
| {"aString":"Number is 182","n":183} | ||
| {"aString":"Number is 184","n":185} | ||
| {"aString":"Number is 186","n":187} | ||
| {"aString":"Number is 188","n":189} | ||
| {"aString":"Number is 190","n":191} | ||
| {"aString":"Number is 192","n":193} | ||
| {"aString":"Number is 194","n":195} | ||
| {"aString":"Number is 196","n":197} | ||
| {"aString":"Number is 198","n":199} |
26 changes: 26 additions & 0 deletions
26
HDFSFormatter/hdfs2.helpers.samples/HDFSJSONFormat/info.xml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| <?xml version="1.0" encoding="UTF-8"?> | ||
| <!-- Copyright (C) 2013-2014, International Business Machines Corporation | ||
| All Rights Reserved | ||
| --> | ||
| <info:toolkitInfoModel xmlns:common="http://www.ibm.com/xmlns/prod/streams/spl/common" xmlns:info="http://www.ibm.com/xmlns/prod/streams/spl/toolkitInfo"> | ||
| <info:identity> | ||
| <info:name>HDFSJSONFormat</info:name> | ||
| <info:description></info:description> | ||
| <info:version>1.0.0</info:version> | ||
| <info:requiredProductVersion>3.0.0</info:requiredProductVersion> | ||
| </info:identity> | ||
| <info:dependencies> | ||
| <info:toolkit> | ||
| <common:name>com.ibm.streams.bigdata</common:name> | ||
| <common:version>2.1.0</common:version> | ||
| </info:toolkit> | ||
| <info:toolkit> | ||
| <common:name>com.ibm.streamsx.json</common:name> | ||
| <common:version>1.0.0</common:version> | ||
| </info:toolkit> | ||
| <info:toolkit> | ||
| <common:name>streamsx.hdfs2.helpers</common:name> | ||
| <common:version>1.0.0</common:version> | ||
| </info:toolkit> | ||
| </info:dependencies> | ||
| </info:toolkitInfoModel> |
97 changes: 97 additions & 0 deletions
97
HDFSFormatter/hdfs2.helpers.samples/HDFSJSONFormat/sample/HDFSJSONFormat.spl
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| /* Copyright (C) 2013-2014, International Business Machines Corporation */ | ||
| /* All Rights Reserved */ | ||
|
|
||
| namespace sample; | ||
| use com.ibm.streamsx.json::JSONToTuple; | ||
| use com.ibm.streamsx.json::TupleToJSON; | ||
| use com.ibm.streamsx.hdfs2.helpers::HDFSReadFormat; | ||
| use com.ibm.streamsx.hdfs2.helpers::HDFSFormatWrite; | ||
|
|
||
| /** Wrapper around JSONToTuple so that it accepts a type argument. | ||
| * The JSONToTuple operator does not need a outType argument, | ||
| * as it inspects it output stream type. However, the decoder | ||
| * passed into the composite needs to accept an output type, | ||
| * so we wrap the operator. | ||
| * @input inStream Stream of strings in JSON format. | ||
| * @output outStream Stream of tuples corresponding to JSON input stream. | ||
| * @param outType type of the output. | ||
| */ | ||
| composite JSONToTupleWrapper (input inStream; | ||
| output outStream) { | ||
| param | ||
| type $outType; | ||
| graph | ||
| stream<$outType> outStream = JSONToTuple(inStream) { | ||
| } | ||
| } | ||
|
|
||
| /** Composite demonstrating the use of the JSON with the formatters. | ||
| * To setup for running this composite | ||
| * - Get the streamsx.json toolkit from IBMStreams | ||
| * - Copy data/inputForJSONRead.txt to hadoopHelperSamples in HDFS. | ||
| * | ||
| * This example shows how to use your own formatters to convert tuples | ||
| * to and from rstrings. The format we use in for this example is the | ||
| * JSON format. | ||
| * | ||
| * After running, in the hadoopBaseDir, you'll see the file jsonWriteTest.txt | ||
| * which contains a json format of the tuples from the beacon operator. | ||
| * <code> | ||
| * \\\{"aString":"Number is 0","n":0\\\} | ||
| * \\\{"aString":"Number is 1","n":1\\\} | ||
| * \\\{"aString":"Number is 2","n":2\\\} | ||
| * ... | ||
| * </code> | ||
| * | ||
| * In the local data directory, you'll see jsonReadTest.csv, which contains csv | ||
| * versions of the tuples read from JSON format. | ||
| * | ||
| * <code> | ||
| * "Number is 0",1 | ||
| * "Number is 2",3 | ||
| * "Number is 4",5 | ||
| * </code> | ||
| */ | ||
| composite JSONMain { | ||
| param | ||
|
|
||
| // Modify this parameter if you wish to use a different | ||
| // HDFS for the reading and writings. | ||
| expression<rstring> $hadoopBaseDir: "hdfsHelperSamples_JSON/"; | ||
| graph | ||
|
|
||
| // Generate some tuples. | ||
| stream<rstring aString, int32 n> testStream = Beacon() { | ||
| param | ||
| iterations: 100; | ||
| output testStream: | ||
| aString = "Number is "+(rstring)IterationCount(), | ||
| n = (int32)IterationCount(); | ||
| } | ||
|
|
||
| // Write those tuples to HDFS. After the application has run, | ||
| // verify the tuples are here. | ||
| () as sink = HDFSFormatWrite(testStream) { | ||
| param | ||
| file: $hadoopBaseDir+"jsonWriteTest.txt"; | ||
| formatter: TupleToJSON; | ||
| } | ||
|
|
||
| // The tuples are stored on HDFS in JSON format. This composite | ||
| // expands to two operators--one that reads lines, and a second | ||
| // that converts the lines to tuples. | ||
| stream<rstring aString, int64 n> readTest = HDFSReadFormat() { | ||
| param | ||
| file: $hadoopBaseDir+"inputForJSONRead.txt"; | ||
| decoder: JSONToTupleWrapper; | ||
| streamType: tuple<rstring aString,int64 n>; | ||
| } | ||
|
|
||
| // Write the read tuples locally, so you can verify them. | ||
| () as JsonReadTest = FileSink(readTest) { | ||
| param | ||
| file: "jsonReadTest.cvs"; | ||
| format: csv; | ||
| } | ||
|
|
||
| } |
29 changes: 29 additions & 0 deletions
29
HDFSFormatter/hdfs2.helpers.samples/ReadFormatWrite/.project
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| <?xml version="1.0" encoding="UTF-8"?> | ||
| <projectDescription> | ||
| <name>ReadFormatWrite</name> | ||
| <comment></comment> | ||
| <projects> | ||
| </projects> | ||
| <buildSpec> | ||
| <buildCommand> | ||
| <name>org.eclipse.jdt.core.javabuilder</name> | ||
| <arguments> | ||
| </arguments> | ||
| </buildCommand> | ||
| <buildCommand> | ||
| <name>com.ibm.streams.studio.splproject.builder.SPLProjectBuilder</name> | ||
| <arguments> | ||
| </arguments> | ||
| </buildCommand> | ||
| <buildCommand> | ||
| <name>org.eclipse.xtext.ui.shared.xtextBuilder</name> | ||
| <arguments> | ||
| </arguments> | ||
| </buildCommand> | ||
| </buildSpec> | ||
| <natures> | ||
| <nature>com.ibm.streams.studio.splproject.SPLProjectNature</nature> | ||
| <nature>org.eclipse.xtext.ui.shared.xtextNature</nature> | ||
| <nature>org.eclipse.jdt.core.javanature</nature> | ||
| </natures> | ||
| </projectDescription> |
11 changes: 11 additions & 0 deletions
11
HDFSFormatter/hdfs2.helpers.samples/ReadFormatWrite/LICENSE.md
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| ##License | ||
|
|
||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this except in compliance with the License. | ||
|
|
||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
|
|
||
| Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
|
|
||
| See the License for the specific language governing permissions and limitations under the License. |
Oops, something went wrong.