Skip to content

No matched method found for class org.apache.kudu.spark.kudu.KuduContext.insertRows #10

@meteorwen

Description

@meteorwen
library(tidyverse)
library(sparklyr)
library(kudusparklyr)
library(devtools)
library(DBI)

options(kudu.version = "1.9.0")
SPARK_HOME <- "/opt/cloudera/parcels/CDH/lib/spark/"
spark_version <- '2.4.0'

conf <- spark_config()
conf$spark.driver.cores <- 2
conf$spark.executor.cores <- 2
conf$spark.executor.instances <- 20
conf$spark.driver.memory = "8G"
conf$spark.driver.memoryOverhead = "8G" 
conf$spark.executor.memory <- "4G"
conf$spark.memory.fraction <- 0.9
conf$spark.yarn.executor.memoryOverhead <- "10g"
conf$spark.driver.maxResultSize <- "8G"
conf$spark.dynamicAllocation.enabled <- "false"   #动态分配
conf$spark.yan.principal <- "[email protected]"
conf$spark.blacklist.enabled <- "false"
conf$spark.yarn.dist.jars <-  "/home/sky/R/x86_64-redhat-linux-gnu-library/3.6/kudusparklyr/java/kudu-spark_2.2-2.11-1.9.0.jar"
conf$spark.repl.local.jars <- "/home/sky/R/x86_64-redhat-linux-gnu-library/3.6/kudusparklyr/java/kudu-spark_2.2-2.11-1.9.0.jar"
conf$spark.yarn.keytab="/home/sky/R/x86_64-redhat-linux-gnu-library/3.6/kudusparklyr/java"

Sys.setenv(JAVA_HOME="/usr/java/jdk1.8.0_181-cloudera/")
Sys.setenv(SPARK_HOME = '/opt/cloudera/parcels/CDH/lib/spark/')
Sys.setenv(YARN_CONF_DIR = '/opt/cloudera/parcels/CDH/lib/spark/conf/yarn-conf')

sc <- spark_connect(
  master="yarn-client",
  spark_home = Sys.getenv("SPARK_HOME"),
  method = c("shell"),
  app_name = "R_R_R",
  version = "2.4.0",
  config = conf,
  extensions = sparklyr::registered_extensions(),
  packages = NULL,
  scala_version = NULL)

sc$extensions$jars[[1]] <- "/home/sky/R/x86_64-redhat-linux-gnu-library/3.6/kudusparklyr/java/kudu-spark_2.2-2.11-1.6.0.jar"
sc$extensions$jars[[2]] <- "/home/sky/R/x86_64-redhat-linux-gnu-library/3.6/kudusparklyr/java/kudu-spark2_2.11-1.9.0-cdh6.2.1.jar"
sc$extensions$jars[[3]] <- "/home/sky/R/x86_64-redhat-linux-gnu-library/3.6/kudusparklyr/java/kudu-client-1.9.0-cdh6.2.1.jar"
sc$extensions$jars[[4]] <- "/home/sky/R/x86_64-redhat-linux-gnu-library/3.6/kudusparklyr/java/kudu-client-tools-1.9.0-cdh6.2.1.jar"
sc$extensions$jars[[5]] <- "/home/sky/R/x86_64-redhat-linux-gnu-library/3.6/kudusparklyr/java/kudu-spark2-tools_2.11-1.9.0-cdh6.2.1.jar"
kc <- kudusparklyr::kudu_context(sc,"cdh-5:7051,cdh-6:7051")
connection_is_open(kc)


# creat temp spark table

# 时序表
hoursrecord_kudu <- read_kudu_table(kc, "impala::dw_etl.hoursrecord_kudu")
hoursrecord_kudu_sp <- sdf_register(hoursrecord_kudu, "hoursrecord_kudu_sp")

The above code is my deployment connection environment.The kudu table can be accessed normally.
The read_kudu_table function also works fine.
But kudu_insert_rows create_kudu_table
The execution fails.

> iris_sp <- copy_to(kc,select(iris,c(5,1:4)))
> get_kudu_context(kc) %>% 
+   invoke("upsertRows",spark_dataframe(iris_sp),"impala::tmpdb.iris_kd")

错误: java.lang.Exception: No matched method found for class org.apache.kudu.spark.kudu.KuduContext.upsertRows
	at sparklyr.Invoke.invoke(invoke.scala:144)
	at sparklyr.StreamHandler.handleMethodCall(stream.scala:136)
	at sparklyr.StreamHandler.read(stream.scala:61)
	at sparklyr.BackendHandler$$anonfun$channelRead0$1.apply$mcV$sp(handler.scala:58)
	at scala.util.control.Breaks.breakable(Breaks.scala:38)
	at sparklyr.BackendHandler.channelRead0(handler.scala:38)
	at sparklyr.BackendHandler.channelRead0(handler.scala:14)
	at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
	at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1359)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:935)
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:138)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:645)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:580)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:497)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:459)
	at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858)
	at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138)
	at java.lang.Thread.run(Thread.java:748)
> options <- kudu_table_options(kc) %>% 
+   set_num_replicas(3) %>% 
+   add_hash_partitions(list("Species"),4)
> 
> create_kudu_table(kc, 'impala::tmpdb.iris_kudu',sdf_schema(iris_sp),
+                   list("Species","Sepal_Length","Sepal_Width","Petal_Length","Petal_Width"),
+                   options)

错误: org.apache.kudu.client.NonRecoverableException: Bad schema: Nullable key columns are not supported: Species
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:402)
	at org.apache.kudu.client.KuduClient.createTable(KuduClient.java:119)
	at org.apache.kudu.spark.kudu.KuduContext.createTable(KuduContext.scala:219)
	at org.apache.kudu.spark.kudu.KuduContext.createTable(KuduContext.scala:207)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at sparklyr.Invoke.invoke(invoke.scala:147)
	at sparklyr.StreamHandler.handleMethodCall(stream.scala:136)
	at sparklyr.StreamHandler.read(stream.scala:61)
	at sparklyr.BackendHandler$$anonfun$channelRead0$1.apply$mcV$sp(handler.scala:58)
	at scala.util.control.Breaks.breakable(Breaks.scala:38)
	at sparklyr.BackendHandler.channelRead0(handler.scala:38)
	at sparklyr.BackendHandler.channelRead0(handler.scala:14)
	at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
	at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1359)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:935)
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:138)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:645)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:580)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:497)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:459)
	at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858)
	at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138)
	at java.lang.Thread.run(Thread.java:748)
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.RpcProxy.dispatchMasterError(RpcProxy.java:380)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:273)
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:152)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:148)
		at org.apache.kudu.client.Connection.messageReceived(Connection.java:391)
		at org.apache.kudu.shaded.org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
		at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243)
		at org.apache.kudu.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
		at org.apache.kudu.shaded.org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
		at org.apache.kudu.shaded.org.jboss.netty.handler.timeout.ReadTimeoutHandler.messageReceived(ReadTimeoutHandler.java:184)
		at org.apache.kudu.shaded.org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
		at org.apache.kudu.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
		at org.apache.kudu.shaded.org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
		at org.apache.kudu.shaded.org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296)
		at org.apache.kudu.shaded.org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:70)
		at org.apache.kudu.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
		at org.apache.kudu.shaded.org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
		at org.apache.kudu.shaded.org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296)
		at org.apache.kudu.shaded.org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462)
		at org.apache.kudu.shaded.org.jboss.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443)
		at org.apache.kudu.shaded.org.jboss.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303)
		at org.apache.kudu.shaded.org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
		at org.apache.kudu.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
		at org.apache.kudu.shaded.org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559)
		at org.apache.kudu.shaded.org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268)
		at org.apache.kudu.shaded.org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255)
		at org.apache.kudu.shaded.org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88)
		at org.apache.kudu.shaded.org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108)
		at org.apache.kudu.shaded.org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:337)
		at org.apache.kudu.shaded.org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
		at org.apache.kudu.shaded.org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
		at org.apache.kudu.shaded.org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
		at org.apache.kudu.shaded.org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
		at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
		at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
		... 1 more

错误: java.lang.Exception: No matched method found for class org.apache.kudu.spark.kudu.KuduContext.insertRows

This is the main problem, how to solve?
I followed the github Kudusparklyr example, and by the time these functions were executed:
delete_kudu_table
kudu_insert_rows
create_kudu_table

Failure to execute, report these errors.

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions