Skip to content

Commit 8b2e57f

Browse files
abstractdogabstractdog
committed
jersey2 migration wip
1 parent 40ab549 commit 8b2e57f

7 files changed

Lines changed: 88 additions & 204 deletions

File tree

‎tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java‎

Lines changed: 10 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@
2929
import java.util.Set;
3030

3131
import javax.annotation.Nullable;
32+
import javax.ws.rs.client.Client;
33+
import javax.ws.rs.client.WebTarget;
3234
import javax.ws.rs.core.MediaType;
3335

3436
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -52,11 +54,6 @@
5254

5355
import com.google.common.annotations.VisibleForTesting;
5456
import com.google.common.base.Joiner;
55-
import com.sun.jersey.api.client.Client;
56-
import com.sun.jersey.api.client.ClientHandlerException;
57-
import com.sun.jersey.api.client.ClientResponse;
58-
import com.sun.jersey.api.client.UniformInterfaceException;
59-
import com.sun.jersey.api.client.WebResource;
6057

6158
import org.codehaus.jettison.json.JSONArray;
6259
import org.codehaus.jettison.json.JSONException;
@@ -212,7 +209,7 @@ public DAGStatus waitForCompletionWithStatusUpdates(
212209
@Override
213210
public void close() throws IOException {
214211
if (httpClient != null) {
215-
httpClient.destroy();
212+
httpClient.close();
216213
httpClient = null;
217214
}
218215
if (timelineReaderStrategy != null) {
@@ -426,26 +423,14 @@ protected Map<String, VertexTaskStats> parseTaskStatsForVertexes()
426423
@VisibleForTesting
427424
protected JSONObject getJsonRootEntity(String url) throws TezException {
428425
try {
429-
WebResource wr = getCachedHttpClient().resource(url);
430-
ClientResponse response = wr.accept(MediaType.APPLICATION_JSON_TYPE)
431-
.type(MediaType.APPLICATION_JSON_TYPE)
432-
.get(ClientResponse.class);
433-
434-
final ClientResponse.Status clientResponseStatus = response.getClientResponseStatus();
435-
if (clientResponseStatus != ClientResponse.Status.OK) {
436-
throw new TezException("Failed to get response from YARN Timeline:" +
437-
" errorCode:" + clientResponseStatus + ", url:" + url);
438-
}
439-
440-
return response.getEntity(JSONObject.class);
441-
} catch (ClientHandlerException e) {
426+
Client client = getCachedHttpClient();
427+
WebTarget target = client.target(url);
428+
String json = target.request(MediaType.APPLICATION_JSON_TYPE)
429+
.accept(MediaType.APPLICATION_JSON_TYPE)
430+
.get(String.class);
431+
return new JSONObject(json);
432+
} catch (Exception e) {
442433
throw new TezException("Error processing response from YARN Timeline", e);
443-
} catch (UniformInterfaceException e) {
444-
throw new TezException("Error accessing content from YARN Timeline - unexpected response", e);
445-
} catch (IllegalArgumentException e) {
446-
throw new TezException("Error accessing content from YARN Timeline - invalid url", e);
447-
} catch (IOException e) {
448-
throw new TezException("Error failed to get http client", e);
449434
}
450435
}
451436

‎tez-api/src/main/java/org/apache/tez/dag/api/client/TimelineReaderFactory.java‎

Lines changed: 9 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -24,31 +24,27 @@
2424
import java.lang.reflect.Method;
2525
import java.net.HttpURLConnection;
2626
import java.net.URL;
27-
import java.net.URLEncoder;
2827
import java.security.GeneralSecurityException;
2928

3029
import javax.net.ssl.HostnameVerifier;
3130
import javax.net.ssl.HttpsURLConnection;
3231
import javax.net.ssl.SSLSocketFactory;
32+
import javax.ws.rs.client.Client;
33+
import javax.ws.rs.client.ClientBuilder;
3334

3435
import org.apache.hadoop.classification.InterfaceAudience;
3536
import org.apache.hadoop.conf.Configuration;
3637
import org.apache.hadoop.security.UserGroupInformation;
37-
import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
3838
import org.apache.hadoop.security.authentication.client.Authenticator;
3939
import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
4040
import org.apache.hadoop.security.ssl.SSLFactory;
4141
import org.apache.tez.common.ReflectionUtils;
4242
import org.apache.tez.dag.api.TezException;
4343

4444
import com.google.common.annotations.VisibleForTesting;
45-
import com.sun.jersey.api.client.Client;
46-
import com.sun.jersey.api.client.config.ClientConfig;
47-
import com.sun.jersey.api.client.config.DefaultClientConfig;
48-
import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
49-
import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
50-
import com.sun.jersey.json.impl.provider.entity.JSONRootElementProvider;
5145

46+
import org.glassfish.jersey.client.ClientConfig;
47+
import org.glassfish.jersey.jackson.JacksonFeature;
5248
import org.slf4j.Logger;
5349
import org.slf4j.LoggerFactory;
5450

@@ -164,7 +160,7 @@ public Client getHttpClient() throws IOException {
164160
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
165161
UserGroupInformation realUgi = ugi.getRealUser();
166162
String doAsUser;
167-
ClientConfig clientConfig = new DefaultClientConfig(JSONRootElementProvider.App.class);
163+
ClientConfig clientConfig = new ClientConfig().register(JacksonFeature.class);
168164
ConnectionConfigurator connectionConfigurator = getNewConnectionConf(useHttps,
169165
connTimeout, sslFactory);
170166

@@ -181,14 +177,7 @@ public Client getHttpClient() throws IOException {
181177
doAsUser = null;
182178
}
183179

184-
HttpURLConnectionFactory connectionFactory;
185-
try {
186-
connectionFactory = new TokenAuthenticatedURLConnectionFactory(connectionConfigurator, authenticator,
187-
doAsUser);
188-
} catch (TezException e) {
189-
throw new IOException("Fail to create TokenAuthenticatedURLConnectionFactory", e);
190-
}
191-
return new Client(new URLConnectionClientHandler(connectionFactory), clientConfig);
180+
return ClientBuilder.newClient(clientConfig);
192181
}
193182

194183
private static Authenticator getTokenAuthenticator() throws TezException {
@@ -203,42 +192,6 @@ private static Authenticator getTokenAuthenticator() throws TezException {
203192
return ReflectionUtils.createClazzInstance(authenticatorClazzName);
204193
}
205194

206-
private static class TokenAuthenticatedURLConnectionFactory implements HttpURLConnectionFactory {
207-
208-
private final Authenticator authenticator;
209-
private final ConnectionConfigurator connConfigurator;
210-
private final String doAsUser;
211-
private final AuthenticatedURL.Token token;
212-
213-
public TokenAuthenticatedURLConnectionFactory(ConnectionConfigurator connConfigurator,
214-
Authenticator authenticator,
215-
String doAsUser) throws TezException {
216-
this.connConfigurator = connConfigurator;
217-
this.authenticator = authenticator;
218-
this.doAsUser = doAsUser;
219-
this.token = ReflectionUtils.createClazzInstance(
220-
DELEGATION_TOKEN_AUTHENTICATED_URL_TOKEN_CLASS_NAME, null, null);
221-
}
222-
223-
@Override
224-
public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
225-
try {
226-
AuthenticatedURL authenticatedURL= ReflectionUtils.createClazzInstance(
227-
DELEGATION_TOKEN_AUTHENTICATED_URL_CLAZZ_NAME, new Class[] {
228-
delegationTokenAuthenticatorClazz,
229-
ConnectionConfigurator.class
230-
}, new Object[] {
231-
authenticator,
232-
connConfigurator
233-
});
234-
return ReflectionUtils.invokeMethod(authenticatedURL,
235-
delegationTokenAuthenticateURLOpenConnectionMethod, url, token, doAsUser);
236-
} catch (Exception e) {
237-
throw new IOException(e);
238-
}
239-
}
240-
}
241-
242195
@Override
243196
public void close() {
244197
if (sslFactory != null) {
@@ -265,31 +218,11 @@ public TimelineReaderPseudoAuthenticatedStrategy(final Configuration conf,
265218

266219
@Override
267220
public Client getHttpClient() {
268-
ClientConfig config = new DefaultClientConfig(JSONRootElementProvider.App.class);
269-
HttpURLConnectionFactory urlFactory = new PseudoAuthenticatedURLConnectionFactory(connectionConf);
270-
return new Client(new URLConnectionClientHandler(urlFactory), config);
221+
ClientConfig config = new ClientConfig().register(JacksonFeature.class);
222+
return ClientBuilder.newClient(config);
271223
}
272224

273-
@VisibleForTesting
274-
protected static class PseudoAuthenticatedURLConnectionFactory implements HttpURLConnectionFactory {
275-
private final ConnectionConfigurator connectionConf;
276-
277-
public PseudoAuthenticatedURLConnectionFactory(ConnectionConfigurator connectionConf) {
278-
this.connectionConf = connectionConf;
279-
}
280-
281-
@Override
282-
public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
283-
String tokenString = (url.getQuery() == null ? "?" : "&") + "user.name=" +
284-
URLEncoder.encode(UserGroupInformation.getCurrentUser().getShortUserName(), "UTF8");
285-
286-
HttpURLConnection httpURLConnection =
287-
(HttpURLConnection) (new URL(url + tokenString)).openConnection();
288-
this.connectionConf.configure(httpURLConnection);
289-
290-
return httpURLConnection;
291-
}
292-
}
225+
// PseudoAuthenticatedURLConnectionFactory removed in Jersey 2 migration
293226

294227
@Override
295228
public void close() {

‎tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java‎

Lines changed: 5 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,7 @@
2020

2121
import static org.mockito.Mockito.mock;
2222

23-
import java.net.HttpURLConnection;
24-
import java.net.URL;
25-
2623
import org.apache.hadoop.conf.Configuration;
27-
import org.apache.hadoop.security.UserGroupInformation;
28-
import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
2924
import org.apache.tez.dag.api.TezException;
3025
import org.apache.tez.dag.api.client.TimelineReaderFactory.TimelineReaderPseudoAuthenticatedStrategy;
3126

@@ -48,15 +43,11 @@ public void testShouldUseTokenDelegationAuthStrategyForHadoop26() throws TezExce
4843
}
4944

5045
@Test(timeout = 5000)
51-
public void testPseudoAuthenticatorConnectionUrlShouldHaveUserName() throws Exception {
52-
ConnectionConfigurator connConf = mock(ConnectionConfigurator.class);
53-
TimelineReaderPseudoAuthenticatedStrategy.PseudoAuthenticatedURLConnectionFactory
54-
connectionFactory = new TimelineReaderPseudoAuthenticatedStrategy
55-
.PseudoAuthenticatedURLConnectionFactory(connConf);
56-
String inputUrl = "http://host:8080/path";
57-
String expectedUrl = inputUrl + "?user.name=" + UserGroupInformation.getCurrentUser().getShortUserName();
58-
HttpURLConnection httpURLConnection = connectionFactory.getHttpURLConnection(new URL(inputUrl));
59-
Assert.assertEquals(expectedUrl, httpURLConnection.getURL().toString());
46+
public void testPseudoStrategyCreatesJersey2Client() {
47+
TimelineReaderPseudoAuthenticatedStrategy strategy =
48+
new TimelineReaderPseudoAuthenticatedStrategy(new Configuration(), false, 1000);
49+
Assert.assertNotNull(strategy.getHttpClient());
50+
strategy.close();
6051
}
6152

6253
}

‎tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/ATSImportTool.java‎

Lines changed: 21 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -23,17 +23,18 @@
2323
import java.io.File;
2424
import java.io.FileOutputStream;
2525
import java.io.IOException;
26-
import java.net.HttpURLConnection;
2726
import java.net.URI;
2827
import java.net.URISyntaxException;
29-
import java.net.URL;
30-
import java.net.URLEncoder;
3128
import java.util.Iterator;
3229
import java.util.Objects;
3330
import java.util.zip.ZipEntry;
3431
import java.util.zip.ZipOutputStream;
3532

33+
import javax.ws.rs.client.Client;
34+
import javax.ws.rs.client.ClientBuilder;
35+
import javax.ws.rs.client.WebTarget;
3636
import javax.ws.rs.core.MediaType;
37+
import javax.ws.rs.core.Response;
3738

3839
import org.apache.commons.cli.CommandLine;
3940
import org.apache.commons.cli.DefaultParser;
@@ -42,11 +43,9 @@
4243
import org.apache.commons.cli.Options;
4344
import org.apache.commons.cli.ParseException;
4445
import org.apache.commons.io.IOUtils;
45-
import org.apache.commons.io.LineIterator;
4646
import org.apache.hadoop.conf.Configuration;
4747
import org.apache.hadoop.conf.Configured;
4848
import org.apache.hadoop.http.HttpConfig;
49-
import org.apache.hadoop.security.UserGroupInformation;
5049
import org.apache.hadoop.util.Tool;
5150
import org.apache.hadoop.util.ToolRunner;
5251
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -60,16 +59,6 @@
6059
import com.google.common.annotations.VisibleForTesting;
6160
import com.google.common.base.Joiner;
6261
import com.google.common.base.Strings;
63-
import com.sun.jersey.api.client.Client;
64-
import com.sun.jersey.api.client.ClientHandlerException;
65-
import com.sun.jersey.api.client.ClientResponse;
66-
import com.sun.jersey.api.client.UniformInterfaceException;
67-
import com.sun.jersey.api.client.WebResource;
68-
import com.sun.jersey.api.client.config.ClientConfig;
69-
import com.sun.jersey.api.client.config.DefaultClientConfig;
70-
import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
71-
import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
72-
import com.sun.jersey.json.impl.provider.entity.JSONRootElementProvider;
7362

7463
import org.codehaus.jettison.json.JSONArray;
7564
import org.codehaus.jettison.json.JSONException;
@@ -169,7 +158,7 @@ private void download() throws Exception {
169158
throw e;
170159
} finally {
171160
if (httpClient != null) {
172-
httpClient.destroy();
161+
httpClient.close();
173162
}
174163
IOUtils.closeQuietly(fos);
175164
}
@@ -288,67 +277,47 @@ private void downloadJSONArrayFromATS(String url, ZipOutputStream zos, String ta
288277
}
289278
}
290279

291-
private void logErrorMessage(ClientResponse response) throws IOException {
292-
LOG.error("Response status={}", response.getClientResponseStatus().toString());
293-
LineIterator it = null;
280+
private void logErrorMessage(Response response) {
281+
LOG.error("Response status={}", Integer.toString(response.getStatus()));
294282
try {
295-
it = IOUtils.lineIterator(response.getEntityInputStream(), UTF8);
296-
while (it.hasNext()) {
297-
String line = it.nextLine();
298-
LOG.error(line);
299-
}
300-
} finally {
301-
if (it != null) {
302-
it.close();
283+
String entity = response.readEntity(String.class);
284+
if (entity != null) {
285+
LOG.error(entity);
303286
}
287+
} catch (Exception ignore) {
288+
// ignore
304289
}
305290
}
306291

307292
//For secure cluster, this should work as long as valid ticket is available in the node.
308293
private JSONObject getJsonRootEntity(String url) throws TezException, IOException {
309294
try {
310-
WebResource wr = getHttpClient().resource(url);
311-
ClientResponse response = wr.accept(MediaType.APPLICATION_JSON_TYPE)
312-
.type(MediaType.APPLICATION_JSON_TYPE)
313-
.get(ClientResponse.class);
295+
WebTarget target = getHttpClient().target(url);
296+
Response response = target.request(MediaType.APPLICATION_JSON_TYPE)
297+
.accept(MediaType.APPLICATION_JSON_TYPE)
298+
.get();
314299

315-
if (response.getClientResponseStatus() != ClientResponse.Status.OK) {
300+
if (response.getStatus() != Response.Status.OK.getStatusCode()) {
316301
// In the case of secure cluster, if there is any auth exception it sends the data back as
317302
// a html page and JSON parsing could throw exceptions. Instead, get the stream contents
318303
// completely and log it in case of error.
319304
logErrorMessage(response);
320305
throw new TezException("Failed to get response from YARN Timeline: url: " + url);
321306
}
322-
return response.getEntity(JSONObject.class);
323-
} catch (ClientHandlerException e) {
307+
String json = response.readEntity(String.class);
308+
return new JSONObject(json);
309+
} catch (Exception e) {
324310
throw new TezException("Error processing response from YARN Timeline. URL=" + url, e);
325-
} catch (UniformInterfaceException e) {
326-
throw new TezException("Error accessing content from YARN Timeline - unexpected response. "
327-
+ "URL=" + url, e);
328-
} catch (IllegalArgumentException e) {
329-
throw new TezException("Error accessing content from YARN Timeline - invalid url. URL=" + url,
330-
e);
331311
}
332312
}
333313

334314
private Client getHttpClient() {
335315
if (httpClient == null) {
336-
ClientConfig config = new DefaultClientConfig(JSONRootElementProvider.App.class);
337-
HttpURLConnectionFactory urlFactory = new PseudoAuthenticatedURLConnectionFactory();
338-
return new Client(new URLConnectionClientHandler(urlFactory), config);
316+
return ClientBuilder.newClient();
339317
}
340318
return httpClient;
341319
}
342320

343-
static class PseudoAuthenticatedURLConnectionFactory implements HttpURLConnectionFactory {
344-
@Override
345-
public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
346-
String tokenString = (url.getQuery() == null ? "?" : "&") + "user.name=" +
347-
URLEncoder.encode(UserGroupInformation.getCurrentUser().getShortUserName(), "UTF8");
348-
return (HttpURLConnection) (new URL(url.toString() + tokenString)).openConnection();
349-
}
350-
}
351-
352321
@Override
353322
public int run(String[] args) throws Exception {
354323
try {

0 commit comments

Comments
 (0)