Add logging for out-of-memory debugging

main.cpp

@@ -2041,7 +2041,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
 			vertex_readers.push_back(readers[i].vertexfile);
 			rewind(readers[i].vertexfile);
 		}
-		fqsort(vertex_readers, sizeof(vertex), vertexcmp, vertex_out, memsize / 10);
+		fqsort(vertex_readers, sizeof(vertex), vertexcmp, vertex_out, memsize / 10, 0);
 
 		for (size_t i = 0; i < CPUS; i++) {
 			if (fclose(readers[i].vertexfile) != 0) {
@@ -2107,7 +2107,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
 			rewind(readers[i].nodefile);
 		}
 
-		fqsort(node_readers, sizeof(node), nodecmp, node_out, memsize / 10);
+		fqsort(node_readers, sizeof(node), nodecmp, node_out, memsize / 10, 0);
 
 		for (size_t i = 0; i < CPUS; i++) {
 			if (fclose(readers[i].nodefile) != 0) {

sort.cpp

@@ -6,9 +6,10 @@
 
 #define MAX_MEMORY (10 * 1024 * 1024)
 
-void fqsort(std::vector<FILE *> &inputs, size_t width, int (*cmp)(const void *, const void *), FILE *out, size_t mem) {
+void fqsort(std::vector<FILE *> &inputs, size_t width, int (*cmp)(const void *, const void *), FILE *out, size_t mem, size_t depth) {
 	std::string pivot;
 	FILE *fp1, *fp2;
+	size_t first = 0, second = 0;
 
 	{
 		// read some elements into memory to choose a pivot from
@@ -44,7 +45,9 @@ void fqsort(std::vector<FILE *> &inputs, size_t width, int (*cmp)(const void *,
 			}
 		}
 
+		fprintf(stderr, "%zu: sorting %zu bytes from %zu files\n", depth, buf.size(), inputs.size());
 		qsort((void *) buf.c_str(), buf.size() / width, width, cmp);
+		fprintf(stderr, "%zu: sorted\n", depth);
 
 		// If that was everything we have to sort, we are done.
 
@@ -61,6 +64,7 @@ void fqsort(std::vector<FILE *> &inputs, size_t width, int (*cmp)(const void *,
 		// that compare equal. Does it matter?
 
 		size_t pivot_off = width * (buf.size() / width / 2);
+		fprintf(stderr, "pivot off is %zu from %zu\n", pivot_off, buf.size());
 		pivot = std::string(buf, pivot_off, width);
 
 		std::string t1 = "/tmp/sort1.XXXXXX";
@@ -84,10 +88,13 @@ void fqsort(std::vector<FILE *> &inputs, size_t width, int (*cmp)(const void *,
 
 		fwrite((void *) buf.c_str(), sizeof(char), pivot_off, fp1);
 		fwrite((void *) ((char *) buf.c_str() + pivot_off), sizeof(char), buf.size() - pivot_off, fp2);
+		first = pivot_off;
+		second = buf.size() - pivot_off;
 	}
 
 	// read the remaining input into the temporary files
 
+	size_t additional = 0;
 	for (size_t i = 0; i < inputs.size(); i++) {
 		while (true) {
 			std::string element;
@@ -97,14 +104,18 @@ void fqsort(std::vector<FILE *> &inputs, size_t width, int (*cmp)(const void *,
 			if (n == 0) {
 				break;
 			}
+			additional += n;
 
 			if (cmp((void *) element.c_str(), (void *) pivot.c_str()) < 0) {
 				fwrite((void *) element.c_str(), width, 1, fp1);
+				first += width;
 			} else {
 				fwrite((void *) element.c_str(), width, 1, fp2);
+				second += width;
 			}
 		}
 	}
+	fprintf(stderr, "plus %zu additional bytes (total %zu and %zu)\n", additional, first, second);
 
 	// Now sort the sub-ranges into the output.
 
@@ -113,11 +124,11 @@ void fqsort(std::vector<FILE *> &inputs, size_t width, int (*cmp)(const void *,
 
 	std::vector<FILE *> v1;
 	v1.emplace_back(fp1);
-	fqsort(v1, width, cmp, out, mem);
+	fqsort(v1, width, cmp, out, mem, depth + 1);
 	fclose(fp1);
 
 	std::vector<FILE *> v2;
 	v2.emplace_back(fp2);
-	fqsort(v2, width, cmp, out, mem);
+	fqsort(v2, width, cmp, out, mem, depth + 1);
 	fclose(fp2);
 }

sort.hpp

@@ -1,6 +1,6 @@
 #ifndef SORT_HPP
 #define SORT_HPP
 
-void fqsort(std::vector<FILE *> &inputs, size_t width, int (*cmp)(const void *, const void *), FILE *out, size_t mem);
+void fqsort(std::vector<FILE *> &inputs, size_t width, int (*cmp)(const void *, const void *), FILE *out, size_t mem, size_t depth);
 
 #endif

unit.cpp

@@ -51,7 +51,7 @@ TEST_CASE("External quicksort", "fqsort") {
 	unlink(tmpname.c_str());
 	FILE *f = fdopen(fd, "w+b");
 
-	fqsort(inputs, sizeof(int), intcmp, f, 256);
+	fqsort(inputs, sizeof(int), intcmp, f, 256, 1);
 	rewind(f);
 
 	int prev = INT_MIN;