1+ /*
2+ * Licensed to Elasticsearch under one or more contributor
3+ * license agreements. See the NOTICE file distributed with
4+ * this work for additional information regarding copyright
5+ * ownership. Elasticsearch licenses this file to you under
6+ * the Apache License, Version 2.0 (the "License"); you may
7+ * not use this file except in compliance with the License.
8+ * You may obtain a copy of the License at
9+ *
10+ * http://www.apache.org/licenses/LICENSE-2.0
11+ *
12+ * Unless required by applicable law or agreed to in writing,
13+ * software distributed under the License is distributed on an
14+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+ * KIND, either express or implied. See the License for the
16+ * specific language governing permissions and limitations
17+ * under the License.
18+ */
19+
20+ package org .elasticsearch .hadoop .integration .hive ;
21+
22+ import org .apache .hive .service .cli .HiveSQLException ;
23+ import org .elasticsearch .hadoop .QueryTestParams ;
24+ import org .elasticsearch .hadoop .cfg .ConfigurationOptions ;
25+ import org .elasticsearch .hadoop .mr .RestUtils ;
26+ import org .junit .After ;
27+ import org .junit .Before ;
28+ import org .junit .Test ;
29+ import org .junit .runner .RunWith ;
30+ import org .junit .runners .Parameterized ;
31+ import org .junit .runners .Parameterized .Parameters ;
32+
33+ import java .util .ArrayList ;
34+ import java .util .Arrays ;
35+ import java .util .Collection ;
36+ import java .util .List ;
37+
38+ import static org .elasticsearch .hadoop .integration .hive .HiveSuite .provisionEsLib ;
39+ import static org .elasticsearch .hadoop .integration .hive .HiveSuite .server ;
40+ import static org .junit .Assert .assertEquals ;
41+ import static org .junit .Assert .assertTrue ;
42+ import static org .junit .Assert .fail ;
43+
44+ @ SuppressWarnings ("Duplicates" )
45+ @ RunWith (Parameterized .class )
46+ public class AbstractHiveReadJsonTest {
47+
48+ private static int testInstance = 0 ;
49+ private final boolean readMetadata ;
50+
51+ @ Parameters
52+ public static Collection <Object []> queries () {
53+ return QueryTestParams .params ();
54+ }
55+
56+ private final String query ;
57+
58+ public AbstractHiveReadJsonTest (String query , boolean readMetadata ) {
59+ this .query = query ;
60+ this .readMetadata = readMetadata ;
61+ }
62+
63+ @ Before
64+ public void before () throws Exception {
65+ provisionEsLib ();
66+ RestUtils .refresh ("json-hive" );
67+ }
68+
69+ @ After
70+ public void after () throws Exception {
71+ testInstance ++;
72+ HiveSuite .after ();
73+ }
74+
75+ @ Test
76+ public void basicLoad () throws Exception {
77+
78+ String create = "CREATE EXTERNAL TABLE jsonartistsread" + testInstance + " (data INT, garbage INT, garbage2 STRING) "
79+ + tableProps ("json-hive/artists" , "'es.output.json' = 'true'" , "'es.mapping.names'='garbage2:refuse'" );
80+
81+ String select = "SELECT * FROM jsonartistsread" + testInstance ;
82+
83+ server .execute (create );
84+ List <String > result = server .execute (select );
85+ assertTrue ("Hive returned null" , containsNoNull (result ));
86+ assertContains (result , "Marilyn" );
87+ assertContains (result , "last.fm/music/MALICE" );
88+ assertContains (result , "last.fm/serve/252/5872875.jpg" );
89+ }
90+
91+ @ Test
92+ public void basicLoadWithNameMappings () throws Exception {
93+
94+ String create = "CREATE EXTERNAL TABLE jsonartistsread" + testInstance + " (refuse INT, garbage INT, data STRING) "
95+ + tableProps ("json-hive/artists" , "'es.output.json' = 'true'" , "'es.mapping.names'='data:boomSomethingYouWerentExpecting'" );
96+
97+ String select = "SELECT * FROM jsonartistsread" + testInstance ;
98+
99+ server .execute (create );
100+ List <String > result = server .execute (select );
101+ assertTrue ("Hive returned null" , containsNoNull (result ));
102+ assertContains (result , "Marilyn" );
103+ assertContains (result , "last.fm/music/MALICE" );
104+ assertContains (result , "last.fm/serve/252/5872875.jpg" );
105+ }
106+
107+ @ Test (expected = HiveSQLException .class )
108+ public void basicLoadWithNoGoodCandidateField () throws Exception {
109+
110+ String create = "CREATE EXTERNAL TABLE jsonartistsread" + testInstance + " (refuse INT, garbage INT) "
111+ + tableProps ("json-hive/artists" , "'es.output.json' = 'true'" );
112+
113+ String select = "SELECT * FROM jsonartistsread" + testInstance ;
114+
115+ server .execute (create );
116+ server .execute (select );
117+
118+ fail ("Should have broken because there are no String fields in the table schema to place the JSON data." );
119+ }
120+
121+ @ Test
122+ public void testMissingIndex () throws Exception {
123+ String create = "CREATE EXTERNAL TABLE jsonmissingread" + testInstance + " (data STRING) "
124+ + tableProps ("foobar/missing" , "'es.index.read.missing.as.empty' = 'true'" , "'es.output.json' = 'true'" );
125+
126+ String select = "SELECT * FROM jsonmissingread" + testInstance ;
127+
128+ server .execute (create );
129+ List <String > result = server .execute (select );
130+ assertEquals (0 , result .size ());
131+ }
132+
133+ @ Test
134+ public void testParentChild () throws Exception {
135+ String create = "CREATE EXTERNAL TABLE jsonchildread" + testInstance + " (data STRING) "
136+ + tableProps ("json-hive/child" , "'es.index.read.missing.as.empty' = 'true'" , "'es.output.json' = 'true'" );
137+
138+ String select = "SELECT * FROM jsonchildread" + testInstance ;
139+
140+ System .out .println (server .execute (create ));
141+ List <String > result = server .execute (select );
142+ assertTrue ("Hive returned null" , containsNoNull (result ));
143+ assertTrue (result .size () > 1 );
144+ assertContains (result , "Marilyn" );
145+ assertContains (result , "last.fm/music/MALICE" );
146+ assertContains (result , "last.fm/serve/252/2181591.jpg" );
147+ }
148+
149+ private static boolean containsNoNull (List <String > str ) {
150+ for (String string : str ) {
151+ if (string .contains ("NULL" )) {
152+ return false ;
153+ }
154+ }
155+
156+ return true ;
157+ }
158+
159+ private static void assertContains (List <String > str , String content ) {
160+ for (String string : str ) {
161+ if (string .contains (content )) {
162+ return ;
163+ }
164+ }
165+ fail (String .format ("'%s' not found in %s" , content , str ));
166+ }
167+
168+
169+ private String tableProps (String resource , String ... params ) {
170+ List <String > copy = new ArrayList (Arrays .asList (params ));
171+ copy .add ("'" + ConfigurationOptions .ES_READ_METADATA + "'='" + readMetadata + "'" );
172+ return HiveSuite .tableProps (resource , query , copy .toArray (new String [copy .size ()]));
173+ }
174+ }
0 commit comments