Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Consolidate code for getting platform details into one place #311

Merged
merged 1 commit into from
Dec 9, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ C = $(wildcard *.c) $(wildcard *.cpp)
INCLUDES = -I/usr/local/include -I. -Iclipper2/include
LIBS = -L/usr/local/lib

tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o pmtiles_file.o plugin.o read_json.o write_json.o geobuf.o flatgeobuf.o evaluator.o geocsv.o csv.o geojson-loop.o json_logger.o visvalingam.o compression.o clip.o sort.o attribute.o thread.o shared_borders.o clipper2/src/clipper.engine.o
tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o platform.o text.o dirtiles.o pmtiles_file.o plugin.o read_json.o write_json.o geobuf.o flatgeobuf.o evaluator.o geocsv.o csv.o geojson-loop.o json_logger.o visvalingam.o compression.o clip.o sort.o attribute.o thread.o shared_borders.o clipper2/src/clipper.engine.o
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread

tippecanoe-enumerate: enumerate.o
Expand All @@ -68,7 +68,7 @@ tippecanoe-enumerate: enumerate.o
tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o pmtiles_file.o
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3

tile-join: tile-join.o projection.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o pmtiles_file.o clip.o attribute.o thread.o read_json.o clipper2/src/clipper.engine.o
tile-join: tile-join.o platform.o projection.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o pmtiles_file.o clip.o attribute.o thread.o read_json.o clipper2/src/clipper.engine.o
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread

tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o
Expand Down
49 changes: 9 additions & 40 deletions main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
#include <limits.h>
#include <sqlite3.h>
#include <stdarg.h>
#include <sys/resource.h>
#include <pthread.h>
#include <getopt.h>
#include <signal.h>
Expand Down Expand Up @@ -67,6 +66,7 @@
#include "sort.hpp"
#include "attribute.hpp"
#include "thread.hpp"
#include "platform.hpp"

static int low_detail = 12;
static int full_detail = -1;
Expand Down Expand Up @@ -187,7 +187,7 @@ void init_cpus() {
if (TIPPECANOE_MAX_THREADS != NULL) {
CPUS = atoi_require(TIPPECANOE_MAX_THREADS, "TIPPECANOE_MAX_THREADS");
} else {
CPUS = sysconf(_SC_NPROCESSORS_ONLN);
CPUS = get_num_avail_cpus();
}

if (CPUS < 1) {
Expand All @@ -202,13 +202,7 @@ void init_cpus() {
// Round down to a power of 2
CPUS = 1 << (int) (log(CPUS) / log(2));

struct rlimit rl;
if (getrlimit(RLIMIT_NOFILE, &rl) != 0) {
perror("getrlimit");
exit(EXIT_PTHREAD);
} else {
MAX_FILES = rl.rlim_cur;
}
MAX_FILES = get_max_open_files();

// Don't really want too many temporary files, because the file system
// will start to bog down eventually
Expand All @@ -222,7 +216,7 @@ void init_cpus() {
long long fds[MAX_FILES];
long long i;
for (i = 0; i < MAX_FILES; i++) {
fds[i] = open("/dev/null", O_RDONLY | O_CLOEXEC);
fds[i] = open(get_null_device(), O_RDONLY | O_CLOEXEC);
if (fds[i] < 0) {
break;
}
Expand Down Expand Up @@ -899,7 +893,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
std::atomic<long long> indexpos(indexst.st_size);
int bytes = sizeof(struct index);

int page = sysconf(_SC_PAGESIZE);
int page = get_page_size();
// Don't try to sort more than 2GB at once,
// which used to crash Macs and may still
long long max_unit = 2LL * 1024 * 1024 * 1024;
Expand Down Expand Up @@ -1069,31 +1063,6 @@ void prep_drop_states(struct drop_state *ds, int maxzoom, int basezoom, double d
}
}

static size_t calc_memsize() {
size_t mem;

#ifdef __APPLE__
int64_t hw_memsize;
size_t len = sizeof(int64_t);
if (sysctlbyname("hw.memsize", &hw_memsize, &len, NULL, 0) < 0) {
perror("sysctl hw.memsize");
exit(EXIT_MEMORY);
}
mem = hw_memsize;
#else
long long pagesize = sysconf(_SC_PAGESIZE);
long long pages = sysconf(_SC_PHYS_PAGES);
if (pages < 0 || pagesize < 0) {
perror("sysconf _SC_PAGESIZE or _SC_PHYS_PAGES");
exit(EXIT_MEMORY);
}

mem = (long long) pages * pagesize;
#endif

return mem;
}

void radix(std::vector<struct reader> &readers, int nreaders, FILE *geomfile, FILE *indexfile, const char *tmpdir, std::atomic<long long> *geompos, int maxzoom, int basezoom, double droprate, double gamma) {
// Run through the index and geometry for each reader,
// splitting the contents out by index into as many
Expand Down Expand Up @@ -1446,7 +1415,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
size_t dist_count = 0;
double area_sum = 0;

int files_open_before_reading = open("/dev/null", O_RDONLY | O_CLOEXEC);
int files_open_before_reading = open(get_null_device(), O_RDONLY | O_CLOEXEC);
if (files_open_before_reading < 0) {
perror("open /dev/null");
exit(EXIT_OPEN);
Expand Down Expand Up @@ -1887,7 +1856,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
}
}

int files_open_after_reading = open("/dev/null", O_RDONLY | O_CLOEXEC);
int files_open_after_reading = open(get_null_device(), O_RDONLY | O_CLOEXEC);
if (files_open_after_reading < 0) {
perror("open /dev/null");
exit(EXIT_OPEN);
Expand Down Expand Up @@ -3728,7 +3697,7 @@ int main(int argc, char **argv) {

signal(SIGPIPE, SIG_IGN);

files_open_at_start = open("/dev/null", O_RDONLY | O_CLOEXEC);
files_open_at_start = open(get_null_device(), O_RDONLY | O_CLOEXEC);
if (files_open_at_start < 0) {
perror("open /dev/null");
exit(EXIT_OPEN);
Expand Down Expand Up @@ -3878,7 +3847,7 @@ int main(int argc, char **argv) {
muntrace();
#endif

i = open("/dev/null", O_RDONLY | O_CLOEXEC);
i = open(get_null_device(), O_RDONLY | O_CLOEXEC);
// i < files_open_at_start is not an error, because reading from a pipe closes stdin
if (i > files_open_at_start) {
fprintf(stderr, "Internal error: did not close all files: %d\n", i);
Expand Down
56 changes: 56 additions & 0 deletions platform.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
#include "platform.hpp"
#include <cstdlib>
#include <cstdio>
#include <unistd.h>
#include <sys/resource.h>

#if defined(__APPLE__) || defined(__FreeBSD__)
#include <sys/types.h>
#include <sys/sysctl.h>
#include <sys/param.h>
#include <sys/mount.h>
#endif

#include "errors.hpp"

long get_num_avail_cpus() {
return sysconf(_SC_NPROCESSORS_ONLN);
}

long get_page_size() {
return sysconf(_SC_PAGESIZE);
}

size_t calc_memsize() {
size_t mem;

#ifdef __APPLE__
int64_t hw_memsize;
size_t len = sizeof(int64_t);
if (sysctlbyname("hw.memsize", &hw_memsize, &len, NULL, 0) < 0) {
perror("sysctl hw.memsize");
exit(EXIT_MEMORY);
}
mem = hw_memsize;
#else
long long pagesize = sysconf(_SC_PAGESIZE);
long long pages = sysconf(_SC_PHYS_PAGES);
if (pages < 0 || pagesize < 0) {
perror("sysconf _SC_PAGESIZE or _SC_PHYS_PAGES");
exit(EXIT_MEMORY);
}

mem = (long long) pages * pagesize;
#endif

return mem;
}

size_t get_max_open_files() {
struct rlimit rl;
if (getrlimit(RLIMIT_NOFILE, &rl) != 0) {
perror("getrlimit");
exit(EXIT_PTHREAD);
}
return rl.rlim_cur;
}
16 changes: 16 additions & 0 deletions platform.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#ifndef PLATFORM_HPP
#define PLATFORM_HPP

#include <cstddef>

long get_num_avail_cpus();

long get_page_size();

size_t calc_memsize();

size_t get_max_open_files();

constexpr const char *get_null_device() { return "/dev/null"; }

#endif // PLATFORM_HPP
3 changes: 2 additions & 1 deletion tile-join.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
#include "errors.hpp"
#include "geometry.hpp"
#include "thread.hpp"
#include "platform.hpp"

int pk = false;
int pC = false;
Expand Down Expand Up @@ -1199,7 +1200,7 @@ int main(int argc, char **argv) {

struct tileset_reader *readers = NULL;

CPUS = sysconf(_SC_NPROCESSORS_ONLN);
CPUS = get_num_avail_cpus();

const char *TIPPECANOE_MAX_THREADS = getenv("TIPPECANOE_MAX_THREADS");
if (TIPPECANOE_MAX_THREADS != NULL) {
Expand Down