add support for fieldDelimiter

Author: Mark IIH

NEWS.md

@@ -1,6 +1,6 @@
 # bigQueryR 0.3.2.9000
 
-* support `nullMarker`, `maxBadRecords` in upload jobs
+* support `nullMarker`, `maxBadRecords`, `fieldDelimiter` in upload jobs
 * Support BigQuery type `DATE` for R class `Date` data.frame columns (BigQuery type `TIMESTAMP` still default for `POSIXct`columns) (#48)
 * Allow custom user schema for uploads of data.frames (#48)
 * Rename misnamed global functions from `bq_` prefix to `bqr_` prefix

R/jobs.R

@@ -134,7 +134,7 @@ bqr_wait_for_job <- function(job, wait=5){
 #' 
 #' @family BigQuery asynch query functions  
 #' @export
-bqr_get_job <- function(jobId, projectId = bqr_get_global_project()){
+bqr_get_job <- function(jobId = .Last.value, projectId = bqr_get_global_project()){
   check_bq_auth()
 
   if(is.job(jobId)){

R/uploadData.R

@@ -15,6 +15,7 @@
 #' @param maxBadRecords The maximum number of bad records that BigQuery can ignore when running the job
 #' @param allowJaggedRows Whether to allow rows with variable length columns
 #' @param allowQuotedNewlines Whether to allow datasets with quoted new lines
+#' @param fieldDelimiter The separator for fields in a CSV file.  Default is comma - \code{,}
 #' 
 #' @return TRUE if successful, FALSE if not. 
 #' 
@@ -78,7 +79,8 @@ bqr_upload_data <- function(projectId = bqr_get_global_project(),
                             nullMarker = NULL,
                             maxBadRecords = NULL,
                             allowJaggedRows = FALSE,
-                            allowQuotedNewlines = FALSE){
+                            allowQuotedNewlines = FALSE,
+                            fieldDelimiter = ","){
 
 
   assert_that(is.string(projectId),
@@ -88,7 +90,8 @@ bqr_upload_data <- function(projectId = bqr_get_global_project(),
               is.flag(wait),
               is.flag(allowJaggedRows),
               is.flag(allowQuotedNewlines),
-              is.flag(autodetect))
+              is.flag(autodetect),
+              is.string(fieldDelimiter))
   sourceFormat <- match.arg(sourceFormat)
   create <- match.arg(create)
 
@@ -123,7 +126,8 @@ bqr_upload_data <- function(projectId = bqr_get_global_project(),
                 nullMarker = nullMarker,
                 maxBadRecords = maxBadRecords,
                 allowJaggedRows = allowJaggedRows,
-                allowQuotedNewlines = allowQuotedNewlines)
+                allowQuotedNewlines = allowQuotedNewlines,
+                fieldDelimiter = fieldDelimiter)
 
 }
 
@@ -140,7 +144,8 @@ bqr_do_upload <- function(upload_data,
                           nullMarker,
                           maxBadRecords,
                           allowJaggedRows,
-                          allowQuotedNewlines){
+                          allowQuotedNewlines,
+                          fieldDelimiter){
   check_bq_auth()
   UseMethod("bqr_do_upload", upload_data)
 }
@@ -158,7 +163,8 @@ bqr_do_upload.data.frame <- function(upload_data,
                                      nullMarker,
                                      maxBadRecords,
                                      allowJaggedRows,
-                                     allowQuotedNewlines){ 
+                                     allowQuotedNewlines,
+                                     fieldDelimiter){ 
 
   if(!is.null(user_schema)){
     schema <- user_schema
@@ -169,6 +175,7 @@ bqr_do_upload.data.frame <- function(upload_data,
   config <- list(
     configuration = list(
       load = list(
+        fieldDelimiter = fieldDelimiter,
         nullMarker = nullMarker,
         maxBadRecords = maxBadRecords,
         sourceFormat = "CSV",
@@ -272,7 +279,8 @@ bqr_do_upload.character <- function(upload_data,
                                     nullMarker,
                                     maxBadRecords,
                                     allowJaggedRows,
-                                    allowQuotedNewlines){
+                                    allowQuotedNewlines,
+                                    fieldDelimiter){
 
   if(length(upload_data) > 1){
     source_uri <- upload_data
@@ -283,14 +291,12 @@ bqr_do_upload.character <- function(upload_data,
   config <- list(
     configuration = list(
       load = list(
+        fieldDelimiter = fieldDelimiter,
         nullMarker = nullMarker,
         maxBadRecords = maxBadRecords,
         sourceFormat = sourceFormat,
         createDisposition = jsonlite::unbox(create),
         sourceUris = source_uri,
-        # schema = list(
-        #   fields = user_schema
-        # ),
         destinationTable = list(
           projectId = projectId,
           datasetId = datasetId,
@@ -305,12 +311,31 @@ bqr_do_upload.character <- function(upload_data,
 
   ## only provide schema if autodetect is FALSE
   if(!autodetect){
-    config <- config$configuration$schema <- list(
-      fields = user_schema
+    config <- list(
+      configuration = list(
+        load = list(
+          fieldDelimiter = fieldDelimiter,
+          nullMarker = nullMarker,
+          maxBadRecords = maxBadRecords,
+          sourceFormat = sourceFormat,
+          createDisposition = jsonlite::unbox(create),
+          sourceUris = source_uri,
+          schema = list(
+             fields = user_schema
+           ),
+          destinationTable = list(
+            projectId = projectId,
+            datasetId = datasetId,
+            tableId = tableId
+          ),
+          autodetect = autodetect,
+          allowJaggedRows = allowJaggedRows,
+          allowQuotedNewlines = allowQuotedNewlines
+        )
+      )
     )
   }
 
-  
   l <- 
     googleAuthR::gar_api_generator("https://www.googleapis.com/bigquery/v2",
                                    "POST",