added run scripts for easy execution

Author: jexp

generate.sh

@@ -1,3 +1,4 @@
 source ./settings.sh
 
-mvn clean test-compile exec:java -Dexec.mainClass=org.neo4j.batchimport.TestDataGenerator -Dexec.classpathScope=test -Dexec.args=sorted
+mvn clean test-compile exec:java -Dexec.mainClass=org.neo4j.batchimport.TestDataGenerator -Dexec.classpathScope=test \
+-Dexec.args="$1 $2 $3 $4"  | grep -iv '\[\(INFO\|debug\)\]'

import.sh

@@ -0,0 +1,8 @@
+DB=${1-target/graph.db}
+shift
+NODES=${1-nodes.csv}
+shift
+RELS=${1-rels.csv}
+shift
+mvn compile exec:java -Dexec.mainClass="org.neo4j.batchimport.Importer" \
+   -Dexec.args="batch.properties $DB $NODES $RELS $@" | grep -iv '\[\(INFO\|debug\)\]'

import_csv.sh

@@ -5,6 +5,7 @@
     <version>1.9-SNAPSHOT</version>
     <name>Neo4j Batch Importer</name>
     <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>Å
         <neo4j.version>1.9</neo4j.version>
         <license-text.header>GPL-3-header.txt</license-text.header>
     </properties>

pom.xml

@@ -1,2 +1,3 @@
 echo "Run in main directory sh sample/import.sh"
-mvn test-compile exec:java -Dexec.mainClass="org.neo4j.batchimport.Importer" -Dexec.args="sample/batch.properties target/graph.db sample/nodes.csv,sample/nodes2.csv sample/rels.csv"
+mvn test-compile exec:java -Dexec.mainClass="org.neo4j.batchimport.Importer" \
+  -Dexec.args="sample/batch.properties target/graph.db sample/nodes.csv,sample/nodes2.csv sample/rels.csv"

run.sh

@@ -17,6 +17,7 @@
 import java.util.*;
 import java.util.zip.GZIPInputStream;
 
+import static org.neo4j.batchimport.Utils.join;
 import static org.neo4j.index.impl.lucene.LuceneIndexImplementation.EXACT_CONFIG;
 import static org.neo4j.index.impl.lucene.LuceneIndexImplementation.FULLTEXT_CONFIG;
 
@@ -61,7 +62,9 @@ protected BatchInserter createBatchInserter(File graphDb, Config config) {
     // todo nodes and rels-files in config
     // todo graphdb in config
     public static void main(String[] args) throws IOException {
-        System.err.println("Usage java -jar batchimport.jar data/dir nodes.csv relationships.csv [node_index node-index-name fulltext|exact nodes_index.csv rel_index rel-index-name fulltext|exact rels_index.csv ....]");
+        System.err.println("Usage: Importer data/dir nodes.csv relationships.csv [node_index node-index-name fulltext|exact nodes_index.csv rel_index rel-index-name fulltext|exact rels_index.csv ....]");
+        System.err.println("Using: Importer "+join(args," "));
+        System.err.println();
 
         final Config config = Config.convertArgumentsToConfig(args);
 

sample/import.sh

@@ -58,4 +58,11 @@ private static String formatNode(NodeRecord record) {
         return String.format("Node[%d] -> %d, .%d", record.getId(), record.getNextRel(), record.getNextProp());
     }
 
+    static String join(String[] types, String delim) {
+        StringBuilder sb =new StringBuilder();
+        for (String type : types) {
+            sb.append(type).append(delim);
+        }
+        return sb.substring(0, sb.length() - delim.length());
+    }
 }

settings.sh

@@ -14,55 +14,65 @@
 @Ignore
 public class TestDataGenerator {
 
-    private static int NODES = 1  * 1000 * 1000;
+    private static final int NODES = 1  * 1000 * 1000;
     private static final int RELS_PER_NODE = 50;
     private static final String[] TYPES = {"ONE","TWO","THREE","FOUR","FIVE","SIX","SEVEN","EIGHT","NINE","TEN"};
-    public static final int NUM_TYPES = 10;
 
     public static void main(String...args) throws IOException {
+        System.out.println("Usage: TestDataGenerator NODES RELS_PER_NODE TYPE1,TYPE2,TYPE3 sorted");
         long relCount=0, time = System.currentTimeMillis();
+
+        int nodes = args.length > 0 ? Integer.parseInt(args[0]) : NODES;
+        int relsPerNode = args.length > 1 ? Integer.parseInt(args[1]) : RELS_PER_NODE;
+        String[] types = args.length > 2 ? args[2].split(",") : TYPES;
+        final boolean sorted = args.length > 0 && args[args.length-1].equalsIgnoreCase("sorted");
+        System.out.println("Using: TestDataGenerator "+nodes+" "+relsPerNode+" "+ Utils.join(types, ",")+" "+(sorted?"sorted":""));
+
         BufferedWriter nodeFile = new BufferedWriter(new FileWriter("nodes.csv"));
         nodeFile.write("Node\tRels\tProperty\tCounter:int\n");
         BufferedWriter relFile = new BufferedWriter(new FileWriter("rels.csv"));
         relFile.write("Start\tEnde\tType\tProperty\tCounter:long\n");
-        final boolean sorted = args.length > 0 && args[0].equalsIgnoreCase("sorted");
+
         if (sorted) {
-            relCount = generateSortedRels(relCount, nodeFile, relFile);
+            relCount = generateSortedRels(relCount, nodeFile, relFile, nodes, relsPerNode, types);
         } else {
-            relCount = generateRandomRels(relCount, nodeFile, relFile);
+            relCount = generateRandomRels(relCount, nodeFile, relFile, nodes, relsPerNode, types);
         }
         nodeFile.close();
         relFile.close();
-        System.out.println("Creating " + NODES + " Nodes and " + relCount + (sorted? " sorted " : "") + " Relationships took " + ((System.currentTimeMillis() - time) / 1000) + " seconds.");
+        long seconds = (System.currentTimeMillis() - time) / 1000;
+        System.out.println("Creating " + nodes + " Nodes and " + relCount + (sorted? " sorted " : "") + " Relationships took " + seconds + " seconds.");
     }
 
-    private static long generateRandomRels(long relCount, BufferedWriter nodeFile, BufferedWriter relFile) throws IOException {
+    private static long generateRandomRels(long relCount, BufferedWriter nodeFile, BufferedWriter relFile, int nodes, int relsPerNode, String[] types) throws IOException {
         Random rnd = new Random();
-        for (int node = 0; node < NODES; node++) {
-            final int rels = rnd.nextInt(RELS_PER_NODE);
+        int numTypes = types.length;
+        for (int node = 0; node < nodes; node++) {
+            final int rels = rnd.nextInt(relsPerNode);
             nodeFile.write(node+"\t"+rels+"\tTEST\t"+node+"\n");
             for (int rel = rels; rel >= 0; rel--) {
                 relCount++;
-                final int node1 = rnd.nextInt(NODES);
-                final int node2 = rnd.nextInt(NODES);
-                relFile.write(node1 + "\t" + node2 + "\t" + TYPES[rel % NUM_TYPES] + "\t" + "Property"+"\t" + relCount+ "\n");
+                final int node1 = rnd.nextInt(nodes);
+                final int node2 = rnd.nextInt(nodes);
+                relFile.write(node1 + "\t" + node2 + "\t" + types[rel % numTypes] + "\t" + "Property"+"\t" + relCount+ "\n");
             }
         }
         return relCount;
     }
-    private static long generateSortedRels(long relCount, BufferedWriter nodeFile, BufferedWriter relFile) throws IOException {
+    private static long generateSortedRels(long relCount, BufferedWriter nodeFile, BufferedWriter relFile, int nodes, int relsPerNode, String[] types) throws IOException {
         Random rnd = new Random();
-        for (int node = 0; node < NODES; node++) {
-            final int rels = rnd.nextInt(RELS_PER_NODE);
+        int numTypes = types.length;
+        for (int node = 0; node < nodes; node++) {
+            final int rels = rnd.nextInt(relsPerNode);
             nodeFile.write(node+"\t"+rels+"\tTEST\t"+node+"\n");
             for (int rel = rels; rel >= 0; rel--) {
                 relCount++;
-                final int target = node + rnd.nextInt(NODES-node);
+                final int target = node + rnd.nextInt(nodes -node);
                 final boolean outgoing = rnd.nextBoolean();
                 if (outgoing) {
-                    relFile.write(node + "\t" + target + "\t" + TYPES[rel % NUM_TYPES] + "\t" + "Property"+"\t" + relCount+ "\n");
+                    relFile.write(node + "\t" + target + "\t" + types[rel % numTypes] + "\t" + "Property"+"\t" + relCount+ "\n");
                 } else {
-                    relFile.write(target + "\t" + node + "\t" + TYPES[rel % NUM_TYPES] + "\t" + "Property"+"\t" + relCount+ "\n");
+                    relFile.write(target + "\t" + node + "\t" + types[rel % numTypes] + "\t" + "Property"+"\t" + relCount+ "\n");
                 }
             }
         }