Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions DESCRIPTION
Original file line number Diff line number Diff line change
@@ -1,15 +1,17 @@
Package: kudusparklyr
Encoding: UTF-8
Type: Package
Title: Sparklyr extension for Kudu support
Version: 0.1.0
Date: 2016-08-11
Author: Brandon Kvarda
Maintainer: Brandon Kvarda <[email protected]>
Version: 0.1.1
Date: 2019-11-19
Authors@R: c(person("Brandon", "Kvarda", email = "[email protected]",
role = c("aut","cre")), person("Gerhard", "Nachtmann",
email = "[email protected]", role = "ctb"))
Depends:
R (>= 3.0.0)
Imports:
sparklyr,
magrittr
Description: Sparklyr extension for Kudu support
License: MIT
RoxygenNote: 6.0.1
Description: Sparklyr extension for Kudu support.
License: MIT + file LICENSE
RoxygenNote: 6.1.1
2 changes: 2 additions & 0 deletions LICENSE
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
YEAR: 2016--2019
# COPYRIGHT HOLDER: <Name of the copyright holder>
57 changes: 46 additions & 11 deletions R/createtableoptions.R
Original file line number Diff line number Diff line change
@@ -1,26 +1,61 @@

#' @title kudu_table_options
#' @description invoke kudu_table_options
#'
#' @param sc spark connection
#'
#' @import magrittr
#' @export
kudu_table_options <- function(sc){
opts <- invoke_new(sc,"org.apache.kudu.client.CreateTableOptions")
opts <- invoke_new(sc,
"org.apache.kudu.client.CreateTableOptions")
opts
}

#' @title set_num_replicas
#' @description set number of replicas
#'
#' @param opts options
#' @param num_replicas integer containing the number of replicas
#' @export
set_num_replicas <- function(opts,num_replicas){
opts %>% invoke("setNumReplicas",as.integer(num_replicas))
set_num_replicas <- function(opts, num_replicas){
opts %>% invoke("setNumReplicas", as.integer(num_replicas))
}

#' @title add_hash_partitions
#' @description add hash partitions to kudu table
#'
#' @param opts additional options
#' @param columns columns for partitioning
#' @param buckets buckets
#' @param seed seed
#'
#' @export
add_hash_partitions <- function(opts,columns,buckets,seed = 0){
cols <- invoke_new(sc,"java.util.ArrayList")
add_hash_partitions <- function(opts, columns, buckets,
seed = 0){
cols <- invoke_new(sc, "java.util.ArrayList")
### probably sc can be exchanged by opts
### or spark_connection(opts)? NOTE no visible binding ...
for(item in columns){
cols %>% invoke("add",item)
cols %>% invoke("add", item)
}
opts %>% invoke("addHashPartitions",cols,as.integer(buckets),as.integer(seed))
opts %>% invoke("addHashPartitions", cols, as.integer(buckets),
as.integer(seed))
}


#' @title set_range_partition_columns
#' @description set range partition columns for kudu table
#'
#' @param opts additional options
#' @param columns columns
#'
#' @export
set_range_partition_columns <- function(opts,columns){
set_range_partition_columns <- function(opts, columns){
cols <- invoke_new(sc, "java.util.ArrayList")
### probably sc can be exchanged by opts
### or spark_connection(opts)? NOTE no visible binding ...
for(item in columns){
cols %>% invoke("add",item)
cols %>% invoke("add", item)
}
opts %>% invoke("setRangePartitionColumns",columns)
opts %>% invoke("setRangePartitionColumns", columns)
}
11 changes: 5 additions & 6 deletions R/dependencies.R
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
spark_dependencies <- function(spark_version, scala_version, ...) {

spark_dependencies <- function(spark_version, scala_version,
...) {
kudu_version <- getOption("kudu.version", default = "1.3.1")

sparklyr::spark_dependency(
jars = c(
system.file(
sprintf("java/kudu-spark_%s-%s-%s.jar", spark_version, scala_version, kudu_version),
sprintf("java/kudu-spark_%s-%s-%s.jar", spark_version,
scala_version, kudu_version),
package = "kudusparklyr"
)
),
packages = c(
)
packages = c()
)
}

Expand Down
Loading