From 90f460c4d4d115c75c9c28fe3fa26cc3528f8481 Mon Sep 17 00:00:00 2001 From: Benamar BELARBI Date: Fri, 6 Oct 2017 15:57:20 +0200 Subject: [PATCH 1/2] Update gaeDataExport.py There is a bug, sometimes columns are missing here is the fix --- gaeDataExport.py | 52 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/gaeDataExport.py b/gaeDataExport.py index 303cc91..c50a93e 100644 --- a/gaeDataExport.py +++ b/gaeDataExport.py @@ -2,6 +2,7 @@ # # Reads data from a Google App Engine backup and exports it to a CSV file +# kill -TERM -$(pgrep -o python) # import sys @@ -142,21 +143,44 @@ def process(output_dir, time_capture, table_tuple, writeFn): with open(join(output_dir, table_name + '.csv'), 'w') as write_file: write_file = csv.writer(write_file) # read files, process, and write + #print 'filenames',filenames + print "______________________ extract for ", table_name, ' ______________' + header_list = [] + display_header_list = ['key'] for filename in filenames: path = join(root, filename) - with open(path, 'r') as raw_file: - reader = records.RecordsReader(raw_file) - for record in reader: - entity_proto = entity_pb.EntityProto(contents=record) - entity = datastore.Entity.FromPb(entity_proto) - if header_list is None: - header_list = parseHeaderFields(entity) - display_header_list = ['key'] - display_header_list.extend(header_list) - writeFn(write_file, entity, display_header_list) - csv_row = entity2csvRow(header_list, entity) - writeFn(write_file, entity, csv_row) - count+=1 + if table_name == 'palette': + with open(path, 'r') as raw_file: + reader1 = records.RecordsReader(raw_file) + for record in reader1: + entity_proto1 = entity_pb.EntityProto(contents=record) + entity1 = datastore.Entity.FromPb(entity_proto1) + headers = parseHeaderFields(entity1) + for header in headers: + if len(headers) > len(header_list): + header_list = headers + print 'header_list=',header_list + display_header_list.extend(header_list) + print "header_list", header_list + writeFn(write_file, None, display_header_list) + for filename in filenames: + path = join(root, filename) + if table_name != 'debug': + with open(path, 'r') as raw_file: + reader = records.RecordsReader(raw_file) + for record in reader: + entity_proto = entity_pb.EntityProto(contents=record) + entity = datastore.Entity.FromPb(entity_proto) + if header_list is None: + header_list = parseHeaderFields(entity) + display_header_list = ['key'] + display_header_list.extend(header_list) + writeFn(write_file, entity, display_header_list) + csv_row = entity2csvRow(header_list, entity) + writeFn(write_file, entity, csv_row) + count+=1 + else: + count+=1 time_capture.end(count) print ' ...converted {count:d} objects of type {obj_type} in {run_time:.2f} seconds | {ms_per_obj:.2f} ms/obj | total time = {total_time:.2f} seconds'.format(count=count, obj_type=table_name, run_time=time_capture.run_time, ms_per_obj=time_capture.ms_per_obj, total_time=time_capture.total_time) return time_capture @@ -180,12 +204,14 @@ def displayResults(results, total_time): types = len(results) total_count = sum([result.count for result in results]) total_run_time = sum([result.run_time for result in results]) + total_count = total_count or 1 avg_ms_per_obj = total_run_time * 1000 / total_count print 'Converted {types:d} types with a total of {count:d} objects | total running time = {run:.2f} seconds | {rate:.2f} ms/obj | total time = {total:.2f} seconds'.format(types=types, count=total_count, run=total_run_time, rate=avg_ms_per_obj, total=total_time) def main(): input_dir, output_dir = getDirs() table_list = listFiles(input_dir) + print "found ",table_list concurrency = cpu_count() print 'Using {0:d} Processes'.format(concurrency) From 398b8c331865a94f30360ae56d65468251feeaf4 Mon Sep 17 00:00:00 2001 From: Benamar BELARBI Date: Fri, 6 Oct 2017 17:20:15 +0200 Subject: [PATCH 2/2] Update gaeDataExport.py without debug log --- gaeDataExport.py | 55 +++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/gaeDataExport.py b/gaeDataExport.py index c50a93e..aa2e18d 100644 --- a/gaeDataExport.py +++ b/gaeDataExport.py @@ -2,6 +2,7 @@ # # Reads data from a Google App Engine backup and exports it to a CSV file +# to stop all process # kill -TERM -$(pgrep -o python) # @@ -144,43 +145,39 @@ def process(output_dir, time_capture, table_tuple, writeFn): write_file = csv.writer(write_file) # read files, process, and write #print 'filenames',filenames - print "______________________ extract for ", table_name, ' ______________' + #print "______________________ extract for ", table_name, ' ______________' header_list = [] display_header_list = ['key'] for filename in filenames: path = join(root, filename) - if table_name == 'palette': - with open(path, 'r') as raw_file: - reader1 = records.RecordsReader(raw_file) - for record in reader1: - entity_proto1 = entity_pb.EntityProto(contents=record) - entity1 = datastore.Entity.FromPb(entity_proto1) - headers = parseHeaderFields(entity1) - for header in headers: - if len(headers) > len(header_list): - header_list = headers - print 'header_list=',header_list + with open(path, 'r') as raw_file: + reader1 = records.RecordsReader(raw_file) + for record in reader1: + entity_proto1 = entity_pb.EntityProto(contents=record) + entity1 = datastore.Entity.FromPb(entity_proto1) + headers = parseHeaderFields(entity1) + for header in headers: + if len(headers) > len(header_list): + header_list = headers + #print 'header_list=',header_list display_header_list.extend(header_list) - print "header_list", header_list + #print "header_list", header_list writeFn(write_file, None, display_header_list) for filename in filenames: path = join(root, filename) - if table_name != 'debug': - with open(path, 'r') as raw_file: - reader = records.RecordsReader(raw_file) - for record in reader: - entity_proto = entity_pb.EntityProto(contents=record) - entity = datastore.Entity.FromPb(entity_proto) - if header_list is None: - header_list = parseHeaderFields(entity) - display_header_list = ['key'] - display_header_list.extend(header_list) - writeFn(write_file, entity, display_header_list) - csv_row = entity2csvRow(header_list, entity) - writeFn(write_file, entity, csv_row) - count+=1 - else: - count+=1 + with open(path, 'r') as raw_file: + reader = records.RecordsReader(raw_file) + for record in reader: + entity_proto = entity_pb.EntityProto(contents=record) + entity = datastore.Entity.FromPb(entity_proto) + if header_list is None: + header_list = parseHeaderFields(entity) + display_header_list = ['key'] + display_header_list.extend(header_list) + writeFn(write_file, entity, display_header_list) + csv_row = entity2csvRow(header_list, entity) + writeFn(write_file, entity, csv_row) + count+=1 time_capture.end(count) print ' ...converted {count:d} objects of type {obj_type} in {run_time:.2f} seconds | {ms_per_obj:.2f} ms/obj | total time = {total_time:.2f} seconds'.format(count=count, obj_type=table_name, run_time=time_capture.run_time, ms_per_obj=time_capture.ms_per_obj, total_time=time_capture.total_time) return time_capture