Check for readline before requiring it in Python

README.md

@@ -19,7 +19,7 @@ This client code allows you to interact with Cloudera Manager to:
 * Upgrade services running on your cluster
 * Access time-series data on resource utilitization for any activity in the system
 * Read logs for all processes in the system as well as stdout and stderr
-* Programmatically configuration all aspects of your deployment
+* Programmatically configure all aspects of your deployment
 * Collect diagnostic data to aid in debugging issues
 * Run distributed commands to manage auto-failover, host decommissioning and more
 * View all events and alerts that have occurred in the system

java/enunciate.xml

@@ -2,18 +2,18 @@
            xsi:noNamespaceSchemaLocation="http://enunciate.codehaus.org/schemas/enunciate-1.27.xsd">
 
   <api-classes>
-    <include pattern="com.cloudera.api.v9.*"/>
+    <include pattern="com.cloudera.api.v10.*"/>
   </api-classes>
 
   <services>
-    <rest defaultRestSubcontext="/api/v9"/>
+    <rest defaultRestSubcontext="/api/v10"/>
   </services>
 
   <modules>
     <docs splashPackage="com.cloudera.api"
           copyright="Cloudera, Inc. All rights reserved."
           docsDir="apidocs"
-          title="Cloudera Manager API v9"
+          title="Cloudera Manager API v10"
           groupRestResources="byPath"
           freemarkerXMLProcessingTemplate="api-docs.fmt"
           css="static/cms/css/api-docs.css"

java/pom.xml

@@ -5,14 +5,14 @@
   <groupId>com.cloudera.api</groupId>
   <artifactId>cloudera-manager-api</artifactId>
   <name>Cloudera Manager API</name>
-  <version>5.3.0</version>
+  <version>5.4.0</version>
 
   <properties>
     <cxf.version>2.7.5</cxf.version>
     <guava.version>14.0</guava.version>
     <jackson2.version>2.1.0</jackson2.version>
     <joda.version>2.1</joda.version>
-    <junit.version>4.8.2</junit.version>
+    <junit.version>4.11</junit.version>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <test.redirectToFile>true</test.redirectToFile>
     <privateClassPath>com.cloudera.api.shaded</privateClassPath>

java/src/main/java/com/cloudera/api/ApiRootResource.java

@@ -24,6 +24,7 @@
 import com.cloudera.api.v7.RootResourceV7;
 import com.cloudera.api.v8.RootResourceV8;
 import com.cloudera.api.v9.RootResourceV9;
+import com.cloudera.api.v10.RootResourceV10;
 
 import javax.ws.rs.Consumes;
 import javax.ws.rs.GET;
@@ -94,6 +95,12 @@ public interface ApiRootResource {
   @Path("/v9")
   RootResourceV9 getRootV9();
 
+  /**
+   * @return The v10 root resource.
+   */
+  @Path("/v10")
+  RootResourceV10 getRootV10();
+
   /**
    * Fetch the current API version supported by the server.
    * <p>

java/src/main/java/com/cloudera/api/ClouderaManagerClientBuilder.java

@@ -18,25 +18,29 @@
 
 import com.fasterxml.jackson.jaxrs.json.JacksonJsonProvider;
 import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.cxf.configuration.jsse.TLSClientParameters;
-import org.apache.cxf.feature.AbstractFeature;
-import org.apache.cxf.feature.LoggingFeature;
-import org.apache.cxf.jaxrs.client.ClientConfiguration;
-import org.apache.cxf.jaxrs.client.JAXRSClientFactoryBean;
-import org.apache.cxf.jaxrs.client.WebClient;
-import org.apache.cxf.transport.http.HTTPConduit;
-import org.apache.cxf.transports.http.configuration.HTTPClientPolicy;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
 
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.security.cert.X509Certificate;
 import java.util.Arrays;
 import java.util.concurrent.TimeUnit;
+
 import javax.net.ssl.TrustManager;
 import javax.net.ssl.X509TrustManager;
 
+import org.apache.cxf.configuration.jsse.TLSClientParameters;
+import org.apache.cxf.feature.AbstractFeature;
+import org.apache.cxf.feature.LoggingFeature;
+import org.apache.cxf.jaxrs.client.ClientConfiguration;
+import org.apache.cxf.jaxrs.client.JAXRSClientFactoryBean;
+import org.apache.cxf.jaxrs.client.WebClient;
+import org.apache.cxf.transport.http.HTTPConduit;
+import org.apache.cxf.transports.http.configuration.HTTPClientPolicy;
+
 public class ClouderaManagerClientBuilder {
   public static final int DEFAULT_TCP_PORT = 7180;
   public static final long DEFAULT_CONNECTION_TIMEOUT = 0;
@@ -61,6 +65,38 @@ public class ClouderaManagerClientBuilder {
   private boolean validateCn = true;
   private TrustManager[] trustManagers = null;
 
+  /**
+   * Cache JAXRSClientFactoryBean per proxyType.
+   *
+   * We need a cache because CXF stores stubs
+   * ({@link org.apache.cxf.jaxrs.model.ClassResourceInfo} objects) as a reference
+   * inside JAXRSServiceFactoryBean, which is composed within instances of
+   * {@link JAXRSClientFactoryBean}.
+   *
+   * This avoids:
+   * - creating a lot of temporaries generated during the proxy creation for
+   * each client
+   *
+   * - ensures that different proxies of the same type actually share the same
+   * ClassResourceInfo, thus reducing aggregate usage
+   *
+   * Also, as a useful side effect, generates proxies with cached proxy types faster.
+   */
+  private static final LoadingCache<Class<?>, JAXRSClientFactoryBean>
+    clientStaticResources =
+      CacheBuilder.newBuilder()
+      .softValues()
+      .build(
+        new CacheLoader<Class<?>, JAXRSClientFactoryBean>(){
+          @Override
+          public JAXRSClientFactoryBean load(Class<?> proxyType) throws Exception {
+            JAXRSClientFactoryBean clientFactoryBean = new JAXRSClientFactoryBean();
+            clientFactoryBean.setResourceClass(proxyType);
+            clientFactoryBean.setProvider(new JacksonJsonProvider(new ApiObjectMapper()));
+            return clientFactoryBean;
+          }
+        });
+
   public ClouderaManagerClientBuilder withBaseURL(URL baseUrl) {
     this.baseUrl = baseUrl;
     return this;
@@ -154,27 +190,40 @@ public void setTrustManagers(TrustManager[] managers) {
     trustManagers = managers;
   }
 
+  /**
+   * Build an ApiRootResource proxy object for communicating with the remote server.
+   * @return an ApiRootResource proxy object
+   */
   public ApiRootResource build() {
     return build(ApiRootResource.class);
   }
 
+  /**
+   * Build a client proxy, for a specific proxy type.
+   * @param proxyType proxy type class
+   * @return client proxy stub
+   */
   protected <T> T build(Class<T> proxyType) {
-    JAXRSClientFactoryBean bean = new JAXRSClientFactoryBean();
-
     String address = generateAddress();
-    boolean isTlsEnabled = address.startsWith("https://");
-    bean.setAddress(address);
-    if (username != null) {
-      bean.setUsername(username);
-      bean.setPassword(password);
-    }
-    if (enableLogging) {
-      bean.setFeatures(Arrays.<AbstractFeature>asList(new LoggingFeature()));
+    T rootResource;
+    // Synchronized on the class to correlate with the scope of clientStaticResources
+    // We want to ensure that the shared bean isn't set concurrently in multiple callers
+    synchronized (ClouderaManagerClientBuilder.class) {
+      JAXRSClientFactoryBean bean =
+          cleanFactory(clientStaticResources.getUnchecked(proxyType));
+      bean.setAddress(address);
+      if (username != null) {
+        bean.setUsername(username);
+        bean.setPassword(password);
+      }
+
+      if (enableLogging) {
+        bean.setFeatures(Arrays.<AbstractFeature>asList(new LoggingFeature()));
+      }
+      rootResource = bean.create(proxyType);
     }
-    bean.setResourceClass(proxyType);
-    bean.setProvider(new JacksonJsonProvider(new ApiObjectMapper()));
 
-    T rootResource = bean.create(proxyType);
+    boolean isTlsEnabled = address.startsWith("https://");
     ClientConfiguration config = WebClient.getConfig(rootResource);
     HTTPConduit conduit = (HTTPConduit) config.getConduit();
     if (isTlsEnabled) {
@@ -197,6 +246,13 @@ else if (trustManagers != null) {
     return rootResource;
   }
 
+  private static JAXRSClientFactoryBean cleanFactory(JAXRSClientFactoryBean bean) {
+    bean.setUsername(null);
+    bean.setPassword(null);
+    bean.setFeatures(Arrays.<AbstractFeature>asList());
+    return bean;
+  }
+
   /**
    * Closes the transport level conduit in the client. Reopening a new
    * connection, requires creating a new client object using the build()
@@ -215,6 +271,21 @@ public static void closeClient(ApiRootResource root) {
     conduit.close();
   }
 
+  /**
+   * Clears any cached resources shared during build operations
+   * across instances of this class.
+   *
+   * This includes shared proxy/stub information, that will be automatically
+   * garbage collected when used heap memory in the JVM
+   * nears max heap memory in the JVM.
+   *
+   * In general, it is unnecessary to invoke this method, unless you are
+   * concerned with reducing used heap memory in the JVM.
+   */
+  public static void clearCachedResources() {
+    clientStaticResources.invalidateAll();
+  }
+
   /** A trust manager that will accept all certificates. */
   private static class AcceptAllTrustManager implements X509TrustManager {
 
@@ -229,7 +300,5 @@ public void checkServerTrusted(X509Certificate[] chain, String authType) {
     public X509Certificate[] getAcceptedIssuers() {
       return null;
     }
-
   }
-
 }

java/src/main/java/com/cloudera/api/model/ApiCluster.java

@@ -23,6 +23,7 @@
 import javax.xml.bind.annotation.XmlElementWrapper;
 import javax.xml.bind.annotation.XmlRootElement;
 import javax.xml.bind.annotation.XmlType;
+
 import java.util.List;
 
 /**

java/src/main/java/com/cloudera/api/model/ApiCollectDiagnosticDataArguments.java

@@ -16,6 +16,9 @@
 
 package com.cloudera.api.model;
 
+import java.util.List;
+
+import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
 
 /**
@@ -31,6 +34,7 @@ public class ApiCollectDiagnosticDataArguments {
   private String ticketNumber;
   private String comments;
   private String clusterName;
+  private List<String> roles;
 
   /**
    * The maximum approximate bundle size of the output file
@@ -115,4 +119,24 @@ public String getClusterName() {
   public void setClusterName(String clusterName) {
     this.clusterName = clusterName;
   }
+
+  /**
+   * List of roles for which to get logs and metrics.
+   *
+   * If set, this restricts the roles for log and metrics collection
+   * to the list specified.
+   *
+   * If empty, the default is to get logs for all roles (in the selected
+   * cluster, if one is selected).
+   *
+   * Introduced in API v10 of the API.
+   */
+  @XmlElement
+  public List<String> getRoles() {
+    return roles;
+  }
+
+  public void setRoles(List<String> roles) {
+    this.roles = roles;
+  }
 }

java/src/main/java/com/cloudera/api/model/ApiHdfsReplicationArguments.java

@@ -44,6 +44,7 @@ public class ApiHdfsReplicationArguments {
   private boolean skipChecksumChecks;
   private Boolean skipTrash;
   private ReplicationStrategy replicationStrategy;
+  private Boolean preserveXAttrs;
 
   /**
    * The strategy for distributing the file replication tasks among the mappers
@@ -224,6 +225,9 @@ public void setPreserveBlockSize(boolean preserveBlockSize) {
   /**
    * Whether to preserve the HDFS owner, group and permissions. Defaults to
    * false.
+   * Starting from V10, it also preserves ACLs. Defaults to null (no preserve).
+   * ACLs is preserved if both clusters enable ACL support, and replication
+   * ignores any ACL related failures.
    */
   @XmlElement
   public boolean getPreservePermissions() {
@@ -284,6 +288,21 @@ public void setReplicationStrategy(ReplicationStrategy replicationStrategy) {
     this.replicationStrategy = replicationStrategy;
   }
 
+  /**
+   * Whether to preserve XAttrs, default to false
+   * This is introduced in V10. To preserve XAttrs, both CDH versions
+   * should be >= 5.2. Replication fails if either cluster does not support
+   * XAttrs.
+   */
+  @XmlElement
+  public Boolean getPreserveXAttrs() {
+    return preserveXAttrs;
+  }
+
+  public void setPreserveXAttrs(Boolean preserveXAttrs) {
+    this.preserveXAttrs = preserveXAttrs;
+  }
+
   @Override
   public String toString() {
     return Objects.toStringHelper(this)
@@ -304,6 +323,7 @@ public String toString() {
                   .add("skipChecksumChecks", skipChecksumChecks)
                   .add("skipTrash", skipTrash)
                   .add("replicationStrategy", replicationStrategy)
+                  .add("preserveXAttrs", preserveXAttrs)
                   .toString();
   }
 
@@ -327,7 +347,8 @@ public boolean equals(Object o) {
         Objects.equal(logPath, other.getLogPath()) &&
         skipChecksumChecks == other.getSkipChecksumChecks() &&
         Objects.equal(skipTrash, other.getSkipTrash()) &&
-        Objects.equal(replicationStrategy, other.getReplicationStrategy()));
+        Objects.equal(replicationStrategy, other.getReplicationStrategy()) &&
+        Objects.equal(preserveXAttrs, other.getPreserveXAttrs()));
   }
 
   @Override
@@ -336,6 +357,7 @@ public int hashCode() {
         mapreduceServiceName, schedulerPoolName, numMaps, dryRun,
         bandwidthPerMap, abortOnError, removeMissingFiles,
         preserveReplicationCount, preserveBlockSize, preservePermissions,
-        logPath, skipChecksumChecks, skipTrash, replicationStrategy);
+        logPath, skipChecksumChecks, skipTrash, replicationStrategy,
+        preserveXAttrs);
   }
 }

java/src/main/java/com/cloudera/api/model/ApiHealthSummary.java

@@ -33,5 +33,5 @@ public enum ApiHealthSummary {
   /** The subject is in concerning health */
   CONCERNING,
   /** The subject is in bad health */
-  BAD
+  BAD;
 }

java/src/main/java/com/cloudera/api/model/ApiHostInstallArguments.java

@@ -141,6 +141,15 @@ public void setPassword(String password) {
   /**
    * The private key to authenticate with the hosts.
    * Specify either this or a password.
+   * <br>
+   * The private key, if specified, needs to be a
+   * standard PEM-encoded key as a single string, with all line breaks
+   * replaced with the line-feed control character '\n'.
+   * <br>
+   * A value will typically look like the following string:
+   * <br>
+   * -----BEGIN RSA PRIVATE KEY-----\n[base-64 encoded key]\n-----END RSA PRIVATE KEY-----
+   * <br>
    */
   @XmlElement
   public String getPrivateKey() {