diff --git a/Makefile b/Makefile index d01eb42b..8679691b 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,7 @@ C = $(wildcard *.c) $(wildcard *.cpp) INCLUDES = -I/usr/local/include -I. -Iclipper2/include LIBS = -L/usr/local/lib -tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o pmtiles_file.o plugin.o read_json.o write_json.o geobuf.o flatgeobuf.o evaluator.o geocsv.o csv.o geojson-loop.o json_logger.o visvalingam.o compression.o clip.o sort.o attribute.o thread.o shared_borders.o clipper2/src/clipper.engine.o +tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o platform.o text.o dirtiles.o pmtiles_file.o plugin.o read_json.o write_json.o geobuf.o flatgeobuf.o evaluator.o geocsv.o csv.o geojson-loop.o json_logger.o visvalingam.o compression.o clip.o sort.o attribute.o thread.o shared_borders.o clipper2/src/clipper.engine.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread tippecanoe-enumerate: enumerate.o @@ -68,7 +68,7 @@ tippecanoe-enumerate: enumerate.o tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o pmtiles_file.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -tile-join: tile-join.o projection.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o pmtiles_file.o clip.o attribute.o thread.o read_json.o clipper2/src/clipper.engine.o +tile-join: tile-join.o platform.o projection.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o pmtiles_file.o clip.o attribute.o thread.o read_json.o clipper2/src/clipper.engine.o $(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o diff --git a/main.cpp b/main.cpp index 0d0a9ed3..aa6c49f2 100644 --- a/main.cpp +++ b/main.cpp @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include @@ -67,6 +66,7 @@ #include "sort.hpp" #include "attribute.hpp" #include "thread.hpp" +#include "platform.hpp" static int low_detail = 12; static int full_detail = -1; @@ -187,7 +187,7 @@ void init_cpus() { if (TIPPECANOE_MAX_THREADS != NULL) { CPUS = atoi_require(TIPPECANOE_MAX_THREADS, "TIPPECANOE_MAX_THREADS"); } else { - CPUS = sysconf(_SC_NPROCESSORS_ONLN); + CPUS = get_num_avail_cpus(); } if (CPUS < 1) { @@ -202,13 +202,7 @@ void init_cpus() { // Round down to a power of 2 CPUS = 1 << (int) (log(CPUS) / log(2)); - struct rlimit rl; - if (getrlimit(RLIMIT_NOFILE, &rl) != 0) { - perror("getrlimit"); - exit(EXIT_PTHREAD); - } else { - MAX_FILES = rl.rlim_cur; - } + MAX_FILES = get_max_open_files(); // Don't really want too many temporary files, because the file system // will start to bog down eventually @@ -222,7 +216,7 @@ void init_cpus() { long long fds[MAX_FILES]; long long i; for (i = 0; i < MAX_FILES; i++) { - fds[i] = open("/dev/null", O_RDONLY | O_CLOEXEC); + fds[i] = open(get_null_device(), O_RDONLY | O_CLOEXEC); if (fds[i] < 0) { break; } @@ -899,7 +893,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split std::atomic indexpos(indexst.st_size); int bytes = sizeof(struct index); - int page = sysconf(_SC_PAGESIZE); + int page = get_page_size(); // Don't try to sort more than 2GB at once, // which used to crash Macs and may still long long max_unit = 2LL * 1024 * 1024 * 1024; @@ -1069,31 +1063,6 @@ void prep_drop_states(struct drop_state *ds, int maxzoom, int basezoom, double d } } -static size_t calc_memsize() { - size_t mem; - -#ifdef __APPLE__ - int64_t hw_memsize; - size_t len = sizeof(int64_t); - if (sysctlbyname("hw.memsize", &hw_memsize, &len, NULL, 0) < 0) { - perror("sysctl hw.memsize"); - exit(EXIT_MEMORY); - } - mem = hw_memsize; -#else - long long pagesize = sysconf(_SC_PAGESIZE); - long long pages = sysconf(_SC_PHYS_PAGES); - if (pages < 0 || pagesize < 0) { - perror("sysconf _SC_PAGESIZE or _SC_PHYS_PAGES"); - exit(EXIT_MEMORY); - } - - mem = (long long) pages * pagesize; -#endif - - return mem; -} - void radix(std::vector &readers, int nreaders, FILE *geomfile, FILE *indexfile, const char *tmpdir, std::atomic *geompos, int maxzoom, int basezoom, double droprate, double gamma) { // Run through the index and geometry for each reader, // splitting the contents out by index into as many @@ -1446,7 +1415,7 @@ std::pair read_input(std::vector &sources, char *fname, i size_t dist_count = 0; double area_sum = 0; - int files_open_before_reading = open("/dev/null", O_RDONLY | O_CLOEXEC); + int files_open_before_reading = open(get_null_device(), O_RDONLY | O_CLOEXEC); if (files_open_before_reading < 0) { perror("open /dev/null"); exit(EXIT_OPEN); @@ -1887,7 +1856,7 @@ std::pair read_input(std::vector &sources, char *fname, i } } - int files_open_after_reading = open("/dev/null", O_RDONLY | O_CLOEXEC); + int files_open_after_reading = open(get_null_device(), O_RDONLY | O_CLOEXEC); if (files_open_after_reading < 0) { perror("open /dev/null"); exit(EXIT_OPEN); @@ -3728,7 +3697,7 @@ int main(int argc, char **argv) { signal(SIGPIPE, SIG_IGN); - files_open_at_start = open("/dev/null", O_RDONLY | O_CLOEXEC); + files_open_at_start = open(get_null_device(), O_RDONLY | O_CLOEXEC); if (files_open_at_start < 0) { perror("open /dev/null"); exit(EXIT_OPEN); @@ -3878,7 +3847,7 @@ int main(int argc, char **argv) { muntrace(); #endif - i = open("/dev/null", O_RDONLY | O_CLOEXEC); + i = open(get_null_device(), O_RDONLY | O_CLOEXEC); // i < files_open_at_start is not an error, because reading from a pipe closes stdin if (i > files_open_at_start) { fprintf(stderr, "Internal error: did not close all files: %d\n", i); diff --git a/platform.cpp b/platform.cpp new file mode 100644 index 00000000..9bf7c3ad --- /dev/null +++ b/platform.cpp @@ -0,0 +1,56 @@ +#include "platform.hpp" +#include +#include +#include +#include + +#if defined(__APPLE__) || defined(__FreeBSD__) +#include +#include +#include +#include +#endif + +#include "errors.hpp" + +long get_num_avail_cpus() { + return sysconf(_SC_NPROCESSORS_ONLN); +} + +long get_page_size() { + return sysconf(_SC_PAGESIZE); +} + +size_t calc_memsize() { + size_t mem; + +#ifdef __APPLE__ + int64_t hw_memsize; + size_t len = sizeof(int64_t); + if (sysctlbyname("hw.memsize", &hw_memsize, &len, NULL, 0) < 0) { + perror("sysctl hw.memsize"); + exit(EXIT_MEMORY); + } + mem = hw_memsize; +#else + long long pagesize = sysconf(_SC_PAGESIZE); + long long pages = sysconf(_SC_PHYS_PAGES); + if (pages < 0 || pagesize < 0) { + perror("sysconf _SC_PAGESIZE or _SC_PHYS_PAGES"); + exit(EXIT_MEMORY); + } + + mem = (long long) pages * pagesize; +#endif + + return mem; +} + +size_t get_max_open_files() { + struct rlimit rl; + if (getrlimit(RLIMIT_NOFILE, &rl) != 0) { + perror("getrlimit"); + exit(EXIT_PTHREAD); + } + return rl.rlim_cur; +} \ No newline at end of file diff --git a/platform.hpp b/platform.hpp new file mode 100644 index 00000000..a62d1cb5 --- /dev/null +++ b/platform.hpp @@ -0,0 +1,16 @@ +#ifndef PLATFORM_HPP +#define PLATFORM_HPP + +#include + +long get_num_avail_cpus(); + +long get_page_size(); + +size_t calc_memsize(); + +size_t get_max_open_files(); + +constexpr const char *get_null_device() { return "/dev/null"; } + +#endif // PLATFORM_HPP \ No newline at end of file diff --git a/tile-join.cpp b/tile-join.cpp index 6d8d2523..ee888959 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -43,6 +43,7 @@ #include "errors.hpp" #include "geometry.hpp" #include "thread.hpp" +#include "platform.hpp" int pk = false; int pC = false; @@ -1199,7 +1200,7 @@ int main(int argc, char **argv) { struct tileset_reader *readers = NULL; - CPUS = sysconf(_SC_NPROCESSORS_ONLN); + CPUS = get_num_avail_cpus(); const char *TIPPECANOE_MAX_THREADS = getenv("TIPPECANOE_MAX_THREADS"); if (TIPPECANOE_MAX_THREADS != NULL) {