diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eff6570ec..6937aa6b8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,11 @@ on: - dev - alpha-dev workflow_dispatch: + inputs: + upload-outputs: + description: 'Whether to upload outputs' + default: false + type: boolean jobs: build-and-test: @@ -58,9 +63,23 @@ jobs: env: SETUPTOOLS_SCM_SUBPROCESS_TIMEOUT: "120" - - name: Run tests - # Disable MacOS for now - we do not yet officially support it and we need to invest a bit - # more efforts into investigating broken LAZ files written by Helios on MacOS. - if: runner.os != 'macOS' + # Do not run on MacOS for now - we do not yet officially support it and we need to invest a bit + # more efforts into investigating broken LAZ files written by Helios on MacOS. + + - name: Run tests (incl. regression tests) + if: runner.os == 'Windows' + run: | + python -m pytest --regression-tests + + - name: Run tests (excl. regression tests) + if: runner.os == 'Linux' run: | python -m pytest + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + if: github.event.inputs.upload-outputs == 'true' + with: + name: test-results + path: output/* + retention-days: 1 diff --git a/data/surveys/demo/tls_arbaro_demo_angular_resolution.xml b/data/surveys/demo/tls_arbaro_demo_angular_resolution.xml index 7d85831ed..abc227474 100644 --- a/data/surveys/demo/tls_arbaro_demo_angular_resolution.xml +++ b/data/surveys/demo/tls_arbaro_demo_angular_resolution.xml @@ -2,7 +2,7 @@ - + diff --git a/environment-dev.yml b/environment-dev.yml index 6a57d1b94..04342dd2e 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -28,3 +28,6 @@ dependencies: - pytest - laspy - lazrs-python + - pandas + - scipy + - pooch diff --git a/pytests/__init__.py b/pytests/__init__.py new file mode 100644 index 000000000..ebf82073c --- /dev/null +++ b/pytests/__init__.py @@ -0,0 +1,4 @@ +import pytest + + +pytest.register_assert_rewrite("pytests.pcloud_utils") diff --git a/pytests/conftest.py b/pytests/conftest.py new file mode 100644 index 000000000..2ca0ac1aa --- /dev/null +++ b/pytests/conftest.py @@ -0,0 +1,61 @@ +import os +import pathlib +import pooch +import pytest +import shutil + + +TEST_DATA_ARCHIVE = "https://github.com/dokempf/helios-test-data/releases/download/2024-08-29/data.tar.gz" +TEST_DATA_CHECKSUM = "df4490f41cd5f9e17fd794429ea7b2fa1f0ad58848b5df44199d24c820cb324b" + + +@pytest.fixture +def regression_data(request): + """ A fixture that ensures the existence of regression data + + Returns the Path to where that data is located after (cached) download + from GitHub. + """ + + if not request.config.getoption("--regression-tests"): + return None + + # Define the cache location + cache = pooch.os_cache("helios") + + # Trigger the download + pooch.retrieve( + TEST_DATA_ARCHIVE, + TEST_DATA_CHECKSUM, + path=cache, + downloader=pooch.HTTPDownloader(timeout=(3, None)), + processor=pooch.Untar(extract_dir="."), + ) + + return cache + + +@pytest.fixture(scope="session") +def output_dir(request): + dir = pathlib.Path(os.getcwd()) / "pytest-output" + + yield dir + + if request.config.getoption("--delete-output"): + shutil.rmtree(dir) + + +def pytest_addoption(parser): + parser.addoption( + "--regression-tests", + action="store_true", + default=False, + help="run regression tests", + ) + + parser.addoption( + "--delete-output", + action="store_true", + default=False, + help="run regression tests", + ) diff --git a/pytests/pcloud_utils.py b/pytests/pcloud_utils.py new file mode 100644 index 000000000..bc9599af6 --- /dev/null +++ b/pytests/pcloud_utils.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python +# -- coding: utf-8 -- + +""" +:author: Alberto M. Esmoris Pena + +Utils for point cloud operations, e.g., point cloud comparison. +""" + +import laspy +import pandas as pd +import numpy as np +from scipy.spatial import KDTree as KDT + +# --- CONSTANTS --- # +# --------------------- # +# Expected names for the features +PCLOUD_FNAME_GPS_TIME = 'gps_time' +PCLOUD_FNAME_REFLECTANCE = 'reflectance' +PCLOUD_FNAME_NIR = 'nir' + + +# --- POINT CLOUD --- # +# ----------------------- # +class PointCloud: + """ + Class representing a point cloud. + + :ivar X: The structure space matrix, i.e., the matrix of point-wise + coordinates. + :vartype X: :class:`np.ndarray` + :ivar fnames: The names for each feature. + :vartype fnames: list of str + :ivar F: The feature space matrix, i.e., the matrix of point-wise features. + :vartype F: :class:`np.ndarray` or None + :ivar y: The vector of classes (represented by integers), if any. + :vartype: :class:`np.ndarray` or None + """ + # --- CONSTRUCTION --- # + # ------------------------ # + def __init__(self, X, fnames=None, F=None, y=None): + self.X = X + self.F = F + self.y = y + self.fnames = fnames + + @staticmethod + def from_las(las, fnames=None, include_classes=False): + """ + Build a point cloud object from the given LAS. + + :param las: The LAS object as generated by laspy. + :param fnames: The names of the features that must be considered. + :param include_classes: Whether to consider the classification (True) + or not (False). + :return: Built point cloud from given LAS. + :rtype: :class:`.PointCloud` + """ + # Extract structure space + scales, offsets = las.header.scales, las.header.offsets + X = np.array([ + las.X * scales[0] + offsets[0], + las.Y * scales[1] + offsets[1], + las.Z * scales[2] + offsets[2], + ]).T + # Extract features + F = None + if fnames is not None: + F = np.array([las[fname] for fname in fnames]).T + # Extract classes + y = None + if include_classes: + y = np.array(las.classification) + # Build point cloud + return PointCloud(X, fnames=fnames, F=F, y=y) + + @staticmethod + def from_las_file(path): + """ + Build a point cloud object from the LAS file at the given path. + + :param path: Path to the LAS file. + """ + return PointCloud.from_las(laspy.read(path)) + + @staticmethod + def from_xyz_file(path, cols, names, sep=' '): + """ + Build a point cloud object from the XYZ/CSV file at the given path. + + :param path: Path to the XYZ/CSV file. + :type path: str + :param cols: The indices of the columns to be selected. + :type cols: list of int + :param names: The name for each selected column. Note that "x", "y", + "z" must be given as the coordinates are necessary to build the + point cloud (features and classes are optional). The name + "classification" is reserved to the classes. Any name that has not + been mentioned before will be understood as a feature. + :type names: list of str + """ + # Read data + P = pd.read_csv(path, usecols=cols, names=names, header=None, sep=sep) + # Extract structure space + X = np.array([P['x'], P['y'], P['z']]).T + # Extract classes + y = P['classification'] if 'classification' in names else None + # Extract features + fnames = [ + name + for name in names if name not in ['x', 'y', 'z', 'classification'] + ] + F = None + if len(fnames) > 0: + F = np.vstack([P[fname] for fname in fnames]).T + else: + fnames = None + # Build point cloud + return PointCloud(X, fnames=fnames, F=F, y=y) + + # --- ASSERT --- # + # ------------------ # + def assert_equals(self, pcloud, eps=1e-5, k=16): + """ + Assert whether two point clouds are equal. + + :param pcloud: The point cloud to compare with. + :param eps: The numeric tolerance threshold. + :param k: How many nearest neighbors must be considered. A high enough + value of k implies that points with the same (x, y, z) but acquired + at different times (i.e., ti != tj) will be properly handled. If + GPSTime. + """ + # Check number of points + assert self.X.shape[0] == pcloud.X.shape[0] + # Check feature names (feature order must also be the same) + check = int(self.fnames is None) + int(pcloud.fnames is None) + assert check != 1 # One has fnames, other does not + if check == 0: # Both have fnames + assert len(self.fnames) == len(pcloud.fnames) + for i, fname in enumerate(self.fnames): + assert fname == pcloud.fnames[i] + # Get neighborhoods + N = self.find_neighborhoods(pcloud, k) + # Compare coordinates + NX = pcloud.X[N] + np.testing.assert_allclose(self.X, NX, atol=eps, rtol=0) + # Compare features + if self.F is not None: + # Check number of features + assert np.all(np.array(self.F.shape) == np.array(pcloud.F.shape)) + # Check numerical differences in the features + NF = pcloud.F[N] + np.testing.assert_allclose(self.F, NF, atol=eps, rtol=0) + # Compare classes + check = int(self.y is None) + int(pcloud.y is None) + assert check != 1 # One has classes, other does not + if check == 0: # Both have classes + Ny = pcloud.y[N] + assert np.any(self.y == Ny) + + # --- UTILS --- # + # ----------------- # + def find_neighborhoods(self, pcloud, k): + """ + Find the nearest neighbor for each points in the self point cloud wrt + the points in the given input point cloud. + """ + # Find GPS time, if available + gps_time_idx = None + if self.fnames is not None: + for i, fname in enumerate(self.fnames): + if fname == PCLOUD_FNAME_GPS_TIME: + gps_time_idx = i + break + # Build KDTree + kdt = KDT(pcloud.X) + # If no GPS time, take closest neighbor + if gps_time_idx is None: + return kdt.query(self.X, 1)[1].tolist() + # If GPS time, untie min distance neighbors with time + else: + t = self.F[:, gps_time_idx] + pcloud_t = pcloud.F[:, gps_time_idx] + D, N = kdt.query(self.X, k) + # NOTE that == is used, abs diff wrt eps could also be used + min_distance_mask = (D.T == np.min(D, axis=1)).T + N = [ni[min_distance_mask[i]] for i, ni in enumerate(N)] + for i, ni in enumerate(N): + jmin = np.argmin(np.abs(pcloud_t[ni]-t[i])) + N[i] = ni[jmin] + return N + + def shuffle(self): + """ + Randomly shuffle the point cloud, i.e., random permutations on the + points. + """ + # Random shuffle on the structure space + indices = np.arange(0, self.X.shape[0]) + np.random.shuffle(indices) + self.X = self.X[indices] + # Random shuffle any other array-like member attribute + for attr in ['F', 'y']: + if getattr(self, attr, None) is not None: + setattr(self, attr, getattr(self, attr)[indices]) + return self # Return the object itself, because fluent :) + + +# --- M A I N --- # +# ------------------- # +# Check the logic when called as an executable script +if __name__ == '__main__': + def clipped_normal(mu, sigma, shape, xmin, xmax): + return np.clip( # Force values to be inside [xmin, xmax] + np.random.normal( # Normal distribution + mu, # Mean + sigma, # Standard deviation + shape # Output dimensionality + ), + xmin, # Min value for clipping + xmax # Max value for clipping + ) + # Test data with no repeated positions and no GPS time + X1 = np.unique(np.random.normal(0, 1, (1024, 3)), axis=0) + fnames1 = [PCLOUD_FNAME_REFLECTANCE, PCLOUD_FNAME_NIR] + F1 = clipped_normal(0, 1, (1024, 2), -3, 3) + y1 = np.random.randint(0, 5, 1024) + pcloud1 = PointCloud(X1, fnames=fnames1, F=F1, y=y1) + + # Test data with no repeated positions and GPS time + fnames2 = fnames1 + [PCLOUD_FNAME_GPS_TIME] + F2 = np.hstack([F1, np.linspace(0, 60, 1024).reshape(-1, 1)]) + pcloud2 = PointCloud(X1, fnames=fnames2, F=F2, y=y1) + + # Test data with repeated positions and no GPS time + X3 = np.random.normal(0, 1, (768, 3)) + X3 = np.vstack([X3, X3[::100], X3[::300]]) + F3 = clipped_normal(0, 1, (X3.shape[0], 1), -3, 3) + y3 = np.random.randint(0, 2, X3.shape[0]) + fnames3 = [PCLOUD_FNAME_REFLECTANCE] + # Ignore F3 in pcloud3 because repeated positions without time will fail + pcloud3 = PointCloud(X3, fnames=fnames3, F=None, y=y3) + + # Test data with repeated positions and GPS time + fnames4 = fnames3 + [PCLOUD_FNAME_GPS_TIME] + F4 = np.hstack([F3, np.linspace(0, 71.3, X3.shape[0]).reshape(-1, 1)]) + pcloud4 = PointCloud(X3, fnames=fnames4, F=F4, y=y3) + + # Test data with no features and no classess at all + X5 = np.random.normal(0, 1, (999, 3)) + pcloud5 = PointCloud(X5, fnames=None, F=None, y=None) + + # Test data in pcloud1 with negligible noise + X6 = X1 + clipped_normal(0, 1e-8, X1.shape, -3e-8, 3e-8) + F6 = F1 + clipped_normal(0, 1e-8, F1.shape, -3e-8, 3e-8) + pcloud6 = PointCloud(X6, fnames=fnames1, F=F6, y=y1).shuffle() + + # Test data in pcloud2 with negligible noise + F7 = np.hstack([F6, np.linspace(0, 60, 1024).reshape(-1, 1)]) + pcloud7 = PointCloud(X6, fnames=fnames2, F=F7, y=y1).shuffle() + + # Run asserts that must be passed + pcloud1.assert_equals(pcloud1) + print('Asserted PointCloud 1') + pcloud2.assert_equals(pcloud2) + print('Asserted PointCloud 2') + pcloud3.assert_equals(pcloud3) + print('Asserted PointCloud 3') + pcloud4.assert_equals(pcloud4) + print('Asserted PointCloud 4') + pcloud5.assert_equals(pcloud5) + print('Asserted PointCloud 5') + pcloud6.assert_equals(pcloud6) + pcloud1.assert_equals(pcloud6) + pcloud6.assert_equals(pcloud1) + print('Asserted PointCloud 6') + pcloud7.assert_equals(pcloud7) + pcloud2.assert_equals(pcloud7) + pcloud7.assert_equals(pcloud2) + print('Asserted PointCloud 7') + # All asserts passed + print('\nAll asserts passed! :)') diff --git a/pytests/test_demo_scenes.py b/pytests/test_demo_scenes.py index 497875c7c..552908786 100644 --- a/pytests/test_demo_scenes.py +++ b/pytests/test_demo_scenes.py @@ -1,7 +1,6 @@ from pyhelios.__main__ import helios_exec import os -import shutil import sys from pathlib import Path import numpy as np @@ -9,20 +8,16 @@ import fnmatch import pyhelios +from . import pcloud_utils as pcu try: import laspy except ImportError: pass -MAX_DIFFERENCE_BYTES = 1024000000 -DELETE_FILES_AFTER = False -WORKING_DIR = os.getcwd() - -def find_playback_dir(survey_path): - playback = Path(WORKING_DIR) / 'output' - with open(Path(WORKING_DIR) / survey_path, 'r') as sf: +def find_playback_dir(survey_path, playback): + with open(survey_path, 'r') as sf: for line in sf: if ' Path: +def run_helios_executable(survey_path: Path, output_dir: Path, options=None) -> Path: if options is None: options = list() - helios_exec([str(survey_path)] + options + ['--rebuildScene', - '--seed', '43', - '-vt', - '-j', '1']) + helios_exec([str(survey_path)] + options + ['--output', str(output_dir), + '--rebuildScene', + '--seed', '43', + '-vt', + '-j', '1']) - return find_playback_dir(survey_path) + return find_playback_dir(survey_path, output_dir) -def run_helios_pyhelios(survey_path: Path, las_output: bool = True, zip_output: bool = False, +def run_helios_pyhelios(survey_path: Path, output: Path, las_output: bool = True, zip_output: bool = False, start_time: str = None, split_by_channel: bool = False, las10: bool = False) -> Path: pyhelios.setDefaultRandomnessGeneratorSeed("43") simB = pyhelios.SimulationBuilder( surveyPath=str(survey_path.absolute()), assetsDir=[str(Path("assets"))], - outputDir=str(Path("output")), + outputDir=str(output), ) simB.setLasOutput(las_output) simB.setLas10(las10) @@ -65,7 +61,7 @@ def run_helios_pyhelios(survey_path: Path, las_output: bool = True, zip_output: sim.start() sim.join() - return find_playback_dir(survey_path) + return find_playback_dir(survey_path, output) def speed_from_traj(trajectory_file): @@ -98,201 +94,217 @@ def mode(arr): @pytest.mark.exe -def test_arbaro_tls_exe(): +def test_arbaro_tls_exe(regression_data, output_dir): dirname_exe = run_helios_executable(Path('data') / 'surveys' / 'demo' / 'tls_arbaro_demo.xml', + output_dir, options=['--lasOutput', - '--gpsStartTime', '2022-01-01 00:00:00']) - eval_arbaro_tls(dirname_exe) + '--gpsStartTime', '2022-01-01 00:00:00', + ]) + eval_arbaro_tls(regression_data, dirname_exe) @pytest.mark.pyh -def test_arbaro_tls_pyh(): +def test_arbaro_tls_pyh(regression_data, output_dir): dirname_pyh = run_helios_pyhelios(Path('data') / 'surveys' / 'demo' / 'tls_arbaro_demo.xml', + output=output_dir, start_time='2022-01-01 00:00:00') - eval_arbaro_tls(dirname_pyh) + eval_arbaro_tls(regression_data, dirname_pyh) -def eval_arbaro_tls(dirname): +def eval_arbaro_tls(regression_data, dirname): assert (dirname / 'leg000_points.las').exists() - assert abs((dirname / 'leg000_points.las').stat().st_size - 22_698_181) < MAX_DIFFERENCE_BYTES assert (dirname / 'leg001_points.las').exists() - assert abs((dirname / 'leg001_points.las').stat().st_size - 14_381_469) < MAX_DIFFERENCE_BYTES + + # ToDo: same approach for trajectory? with open(dirname / 'leg000_trajectory.txt', 'r') as f: line = f.readline() assert line.startswith('1.0000 25.5000 0.0000') - # clean up - if DELETE_FILES_AFTER: - shutil.rmtree(dirname) + + if regression_data: + pcloud0_ref = pcu.PointCloud.from_las_file(regression_data / 'arbaro_tls_leg000_points.las') + pcloud0 = pcu.PointCloud.from_las_file(dirname / 'leg000_points.las') + pcloud0.assert_equals(pcloud0_ref) + + pcloud1_ref = pcu.PointCloud.from_las_file(regression_data / 'arbaro_tls_leg001_points.las') + pcloud1 = pcu.PointCloud.from_las_file(dirname / 'leg001_points.las') + pcloud1.assert_equals(pcloud1_ref) @pytest.mark.exe -def test_tiffloader_als_exe(): +def test_tiffloader_als_exe(regression_data, output_dir): dirname_exe = run_helios_executable(Path('data') / 'test' / 'als_hd_demo_tiff_min.xml', + output_dir, options=['--lasOutput', - '--gpsStartTime', '2022-01-01 00:00:00']) - eval_tiffloader_als(dirname_exe) + '--gpsStartTime', '2022-01-01 00:00:00', + ]) + eval_tiffloader_als(regression_data, dirname_exe) @pytest.mark.pyh -def test_tiffloader_als_pyh(): +def test_tiffloader_als_pyh(regression_data, output_dir): dirname_pyh = run_helios_pyhelios(Path('data') / 'test' / 'als_hd_demo_tiff_min.xml', + output=output_dir, start_time='2022-01-01 00:00:00') - eval_tiffloader_als(dirname_pyh) + eval_tiffloader_als(regression_data, dirname_pyh) -def eval_tiffloader_als(dirname): +def eval_tiffloader_als(regression_data, dirname): assert (dirname / 'leg000_points.las').exists() - assert abs((dirname / 'leg000_points.las').stat().st_size - 53_367) < MAX_DIFFERENCE_BYTES assert (dirname / 'leg001_points.las').exists() - assert abs((dirname / 'leg001_points.las').stat().st_size - 85_557) < MAX_DIFFERENCE_BYTES + assert (dirname / 'leg002_points.las').exists() + + if regression_data: + pcloud0_ref = pcu.PointCloud.from_las_file(regression_data / 'tiffloader_als_leg000_points.las') + pcloud0 = pcu.PointCloud.from_las_file(dirname / 'leg000_points.las') + pcloud0.assert_equals(pcloud0_ref) + pcloud1_ref = pcu.PointCloud.from_las_file(regression_data / 'tiffloader_als_leg001_points.las') + pcloud1 = pcu.PointCloud.from_las_file(dirname / 'leg001_points.las') + pcloud1.assert_equals(pcloud1_ref) + pcloud2_ref = pcu.PointCloud.from_las_file(regression_data / 'tiffloader_als_leg002_points.las') + pcloud2 = pcu.PointCloud.from_las_file(dirname / 'leg002_points.las') + pcloud2.assert_equals(pcloud2_ref) + with open(dirname / 'leg000_trajectory.txt', 'r') as f: next(f) line = f.readline() assert line.startswith('474500.7500 5474500.0000 1500.0000') - # clean up - if DELETE_FILES_AFTER: - shutil.rmtree(dirname) @pytest.mark.exe -def test_detailedVoxels_uls_exe(): +def test_detailedVoxels_uls_exe(regression_data, output_dir): dirname_exe = run_helios_executable(Path('data') / 'test' / 'uls_detailedVoxels_mode_comparison_min.xml', + output_dir, options=['--lasOutput', - '--gpsStartTime', '2022-01-01 00:00:00']) - eval_detailedVoxels_uls(dirname_exe) + '--gpsStartTime', '2022-01-01 00:00:00' + ]) + eval_detailedVoxels_uls(regression_data, dirname_exe) @pytest.mark.pyh -def test_detailedVoxels_uls_pyh(): +def test_detailedVoxels_uls_pyh(regression_data, output_dir): dirname_pyh = run_helios_pyhelios(Path('data') / 'test' / 'uls_detailedVoxels_mode_comparison_min.xml', + output=output_dir, start_time='2022-01-01 00:00:00') - eval_detailedVoxels_uls(dirname_pyh) + eval_detailedVoxels_uls(regression_data, dirname_pyh) -def eval_detailedVoxels_uls(dirname): +def eval_detailedVoxels_uls(regression_data, dirname): assert (dirname / 'leg000_points.las').exists() - assert abs((dirname / 'leg000_points.las').stat().st_size - 460_737) < MAX_DIFFERENCE_BYTES - assert (dirname / 'leg000_trajectory.txt').exists() - assert abs((dirname / 'leg000_trajectory.txt').stat().st_size - 2_541) < MAX_DIFFERENCE_BYTES with open(dirname / 'leg000_trajectory.txt', 'r') as f: for _ in range(6): next(f) line = f.readline() assert line.startswith('-3.0000 -2.1000 50.0000') - # clean up - if DELETE_FILES_AFTER: - shutil.rmtree(dirname) + + if regression_data: + pcloud0_ref = pcu.PointCloud.from_las_file(regression_data / 'detailedVoxels_uls_leg000_points.las') + pcloud0 = pcu.PointCloud.from_las_file(dirname / 'leg000_points.las') + pcloud0.assert_equals(pcloud0_ref) @pytest.mark.exe -def test_xyzVoxels_tls_exe(): +def test_xyzVoxels_tls_exe(regression_data, output_dir): dirname_exe = run_helios_executable(Path('data') / 'surveys' / 'voxels' / 'tls_sphere_xyzloader_normals.xml', + output_dir, options=['--lasOutput', - '--gpsStartTime', '2022-01-01 00:00:00']) - eval_xyzVoxels_tls(dirname_exe) + '--gpsStartTime', '2022-01-01 00:00:00' + ]) + eval_xyzVoxels_tls(regression_data, dirname_exe) @pytest.mark.pyh -def test_xyzVoxels_tls_pyh(): +def test_xyzVoxels_tls_pyh(regression_data, output_dir): dirname_pyh = run_helios_pyhelios(Path('data') / 'surveys' / 'voxels' / 'tls_sphere_xyzloader_normals.xml', + output=output_dir, start_time='2022-01-01 00:00:00') - eval_xyzVoxels_tls(dirname_pyh) + eval_xyzVoxels_tls(regression_data, dirname_pyh) -def eval_xyzVoxels_tls(dirname): +def eval_xyzVoxels_tls(regression_data, dirname): assert (dirname / 'leg000_points.las').exists() - assert abs((dirname / 'leg000_points.las').stat().st_size - 19_352_123) < MAX_DIFFERENCE_BYTES - # clean up - if DELETE_FILES_AFTER: - shutil.rmtree(dirname) + + if regression_data: + pcloud_ref = pcu.PointCloud.from_las_file(regression_data / 'xyzVoxels_tls_leg000_points.las') + pcloud = pcu.PointCloud.from_las_file(dirname / 'leg000_points.las') + pcloud.assert_equals(pcloud_ref) @pytest.mark.exe -def test_interpolated_traj_exe(): +def test_interpolated_traj_exe(regression_data, output_dir): dirname_exe = run_helios_executable(Path('data') / 'surveys' / 'demo' / 'als_interpolated_trajectory.xml', + output_dir, options=['--lasOutput', '--zipOutput', - '--gpsStartTime', '2022-01-01 00:00:00']) - eval_interpolated_traj(dirname_exe) + '--gpsStartTime', '2022-01-01 00:00:00' + ]) + eval_interpolated_traj(regression_data, dirname_exe) @pytest.mark.pyh -def test_interpolated_traj_pyh(): +def test_interpolated_traj_pyh(regression_data, output_dir): dirname_pyh = run_helios_pyhelios(Path('data') / 'surveys' / 'demo' / 'als_interpolated_trajectory.xml', + output=output_dir, zip_output=True, start_time='2022-01-01 00:00:00') - eval_interpolated_traj(dirname_pyh) + eval_interpolated_traj(regression_data, dirname_pyh) -def eval_interpolated_traj(dirname): +def eval_interpolated_traj(regression_data, dirname): assert (dirname / 'leg000_points.laz').exists() - assert (dirname / 'leg000_trajectory.txt').exists() - assert abs((dirname / 'leg000_points.laz').stat().st_size - 850_173) < MAX_DIFFERENCE_BYTES + assert (dirname / 'leg001_points.laz').exists() + assert (dirname / 'leg002_points.laz').exists() + assert (dirname / 'leg003_points.laz').exists() + with open(dirname / 'leg000_trajectory.txt', 'r') as f: for _ in range(3): next(f) line = f.readline() assert line.startswith('13.4766 1.7424 400.0000') - # clean up - if DELETE_FILES_AFTER: - shutil.rmtree(dirname) + + if regression_data: + pcloud0_ref = pcu.PointCloud.from_las_file(regression_data / 'interpolated_traj_leg000_points.laz') + pcloud0 = pcu.PointCloud.from_las_file(dirname / 'leg000_points.laz') + pcloud0.assert_equals(pcloud0_ref) + pcloud1_ref = pcu.PointCloud.from_las_file(regression_data / 'interpolated_traj_leg001_points.laz') + pcloud1 = pcu.PointCloud.from_las_file(dirname / 'leg001_points.laz') + pcloud2_ref = pcu.PointCloud.from_las_file(regression_data / 'interpolated_traj_leg002_points.laz') + pcloud2 = pcu.PointCloud.from_las_file(dirname / 'leg002_points.laz') + pcloud2.assert_equals(pcloud2_ref) + pcloud1.assert_equals(pcloud1_ref) + pcloud3_ref = pcu.PointCloud.from_las_file(regression_data / 'interpolated_traj_leg003_points.laz') + pcloud3 = pcu.PointCloud.from_las_file(dirname / 'leg003_points.laz') + pcloud3.assert_equals(pcloud3_ref) @pytest.mark.skipif("laspy" not in sys.modules, reason="requires the laspy library") @pytest.mark.exe -def test_quadcopter_exe(): +def test_quadcopter_exe(regression_data, output_dir): dirname_exe = run_helios_executable(Path('data') / 'surveys' / 'toyblocks' / 'uls_toyblocks_survey_scene_combo.xml', + output_dir, options=['--lasOutput', '--zipOutput', - '--gpsStartTime', '2022-01-01 00:00:00']) - eval_quadcopter(dirname_exe) + '--gpsStartTime', '2022-01-01 00:00:00', + ]) + eval_quadcopter(regression_data, dirname_exe) @pytest.mark.skipif("laspy" not in sys.modules, reason="requires the laspy library") @pytest.mark.pyh -def test_quadcopter_pyh(): +def test_quadcopter_pyh(regression_data, output_dir): dirname_pyh = run_helios_pyhelios(Path('data') / 'surveys' / 'toyblocks' / 'uls_toyblocks_survey_scene_combo.xml', + output=output_dir, zip_output=True, start_time='2022-01-01 00:00:00') - eval_quadcopter(dirname_pyh) + eval_quadcopter(regression_data, dirname_pyh) -def eval_quadcopter(dirname): +def eval_quadcopter(regression_data, dirname): assert (dirname / 'leg000_points.laz').exists() - assert (dirname / 'leg000_trajectory.txt').exists() - # assert abs( - # (dirname / 'leg000_points.laz').stat().st_size - 1_974_592) < MAX_DIFFERENCE_BYTES - # assert abs( - # (dirname / 'leg002_points.laz').stat().st_size - 2_153_266) < MAX_DIFFERENCE_BYTES - # assert abs( - # (dirname / 'leg004_points.laz').stat().st_size - 3_818_282) < MAX_DIFFERENCE_BYTES - las = laspy.read(dirname / 'leg000_points.laz') - data = np.array([las.x, las.y, las.z]).T - expected = np.array([[-7.00000e+01, -3.35592e+01, 3.73900e-03], - [-7.00000e+01, -3.32781e+01, -5.61000e-04], - [-7.00000e+01, -3.29992e+01, 3.23900e-03], - [-7.00000e+01, -3.27169e+01, -1.16100e-03], - [-7.00000e+01, -3.24399e+01, 1.16390e-02], - [-7.00000e+01, -3.21570e+01, 8.93900e-03], - [-7.00000e+01, -3.18751e+01, 1.09390e-02], - [-7.00000e+01, -3.15897e+01, 4.83900e-03], - [-7.00000e+01, -3.13079e+01, 1.04390e-02], - [-7.00000e+01, -3.10238e+01, 1.14390e-02], - [-7.00000e+01, -3.07325e+01, -5.36100e-03], - [-7.00000e+01, -3.04460e+01, -7.36100e-03], - [-7.00000e+01, -3.01620e+01, -6.61000e-04], - [-7.00000e+01, -2.98794e+01, 1.15390e-02], - [-7.00000e+01, -2.95864e+01, -2.36100e-03], - [-7.00000e+01, -2.93011e+01, 6.13900e-03], - [-7.00000e+01, -2.90174e+01, 2.02390e-02], - [-7.00000e+01, -2.87225e+01, 7.13900e-03], - [-7.00000e+01, -2.84326e+01, 8.83900e-03], - [-7.00000e+01, -2.81384e+01, 1.53900e-03]]) - # atol for numpy assert moved to 1e-3 from 1e-12 due to discrepancies - # between local and remote (GitHub action) results - np.testing.assert_allclose(data[100:120, :], expected, atol=1e-3) + assert (dirname / 'leg001_points.laz').exists() + assert (dirname / 'leg002_points.laz').exists() + assert (dirname / 'leg004_points.laz').exists() assert speed_from_traj(dirname / 'leg000_trajectory.txt') == pytest.approx(10.0, 0.001) assert speed_from_traj(dirname / 'leg002_trajectory.txt') == pytest.approx(7.0, 0.001) assert speed_from_traj(dirname / 'leg004_trajectory.txt') == pytest.approx(4.0, 0.001) @@ -301,49 +313,106 @@ def eval_quadcopter(dirname): next(f) line = f.readline() assert line.startswith('-69.9983 -60.0000 80.0002') - # clean up - if DELETE_FILES_AFTER: - shutil.rmtree(dirname) + + if regression_data: + pcloud0_ref = pcu.PointCloud.from_las_file(regression_data / 'quadcopter_leg000_points.laz') + pcloud0 = pcu.PointCloud.from_las_file(dirname / 'leg000_points.laz') + pcloud0.assert_equals(pcloud0_ref) + pcloud1_ref = pcu.PointCloud.from_las_file(regression_data / 'quadcopter_leg001_points.laz') + pcloud1 = pcu.PointCloud.from_las_file(dirname / 'leg001_points.laz') + pcloud1.assert_equals(pcloud1_ref) + pcloud2_ref = pcu.PointCloud.from_las_file(regression_data / 'quadcopter_leg002_points.laz') + pcloud2 = pcu.PointCloud.from_las_file(dirname / 'leg002_points.laz') + pcloud2.assert_equals(pcloud2_ref) + pcloud3_ref = pcu.PointCloud.from_las_file(regression_data / 'quadcopter_leg004_points.laz') + pcloud3 = pcu.PointCloud.from_las_file(dirname / 'leg004_points.laz') + pcloud3.assert_equals(pcloud3_ref) @pytest.mark.pyh -def test_als_multichannel_pyh(): +def test_als_multichannel_pyh(regression_data, output_dir): dirname_pyh = run_helios_pyhelios(Path('data') / 'surveys' / 'demo' / 'light_als_toyblocks_multiscanner.xml', - zip_output=True) - eval_als_multichannel(dirname_pyh) + output=output_dir, + zip_output=True, + start_time='2022-01-01 00:00:00') + eval_als_multichannel(regression_data, dirname_pyh) @pytest.mark.pyh -def test_als_multichannel_split_pyh(): +def test_als_multichannel_split_pyh(regression_data, output_dir): dirname_pyh = run_helios_pyhelios(Path('data') / 'surveys' / 'demo' / 'light_als_toyblocks_multiscanner.xml', - zip_output=True, split_by_channel=True) - eval_als_multichannel_split(dirname_pyh) + output=output_dir, + zip_output=True, split_by_channel=True, + start_time='2022-01-01 00:00:00') + eval_als_multichannel_split(regression_data, dirname_pyh) @pytest.mark.exe -def test_als_multichannel(): +def test_als_multichannel_exe(regression_data, output_dir): dirname_exe = run_helios_executable(Path('data') / 'surveys' / 'demo' / 'light_als_toyblocks_multiscanner.xml', + output_dir, options=['--lasOutput', - '--zipOutput']) - eval_als_multichannel(dirname_exe) + '--zipOutput', + '--gpsStartTime', '2022-01-01 00:00:00', + ]) + eval_als_multichannel(regression_data, dirname_exe) @pytest.mark.exe -def test_als_multichannel_split(): +def test_als_multichannel_split_exe(regression_data, output_dir): dirname_exe = run_helios_executable(Path('data') / 'surveys' / 'demo' / 'light_als_toyblocks_multiscanner.xml', + output_dir, options=['--lasOutput', '--zipOutput', - '--splitByChannel']) - eval_als_multichannel_split(dirname_exe) + '--gpsStartTime', '2022-01-01 00:00:00', + '--splitByChannel', + ]) + eval_als_multichannel_split(regression_data, dirname_exe) -def eval_als_multichannel(dirname): +def eval_als_multichannel(regression_data, dirname): assert len(fnmatch.filter(os.listdir(dirname), '*.laz')) == 2 + assert (dirname / 'leg000_points.laz').exists() + assert (dirname / 'leg002_points.laz').exists() + + if regression_data: + pcloud0_ref = pcu.PointCloud.from_las_file(regression_data / 'als_multichannel_leg000_points.laz') + pcloud0 = pcu.PointCloud.from_las_file(dirname / 'leg000_points.laz') + pcloud0.assert_equals(pcloud0_ref) + pcloud1_ref = pcu.PointCloud.from_las_file(regression_data / 'als_multichannel_leg002_points.laz') + pcloud1 = pcu.PointCloud.from_las_file(dirname / 'leg002_points.laz') + pcloud1.assert_equals(pcloud1_ref) -def eval_als_multichannel_split(dirname): +def eval_als_multichannel_split(regression_data, dirname): # 2 legs, Livox Mid-100 has 3 channels, so we expect 6 point clouds assert len(fnmatch.filter(os.listdir(dirname), '*.laz')) == 6 + assert (dirname / 'leg000_points_dev0.laz').exists() + assert (dirname / 'leg000_points_dev1.laz').exists() + assert (dirname / 'leg000_points_dev2.laz').exists() + assert (dirname / 'leg002_points_dev0.laz').exists() + assert (dirname / 'leg002_points_dev1.laz').exists() + assert (dirname / 'leg002_points_dev2.laz').exists() + + if regression_data: + pcloud0_0_ref = pcu.PointCloud.from_las_file(regression_data / 'als_multichannel_split_leg000_points_dev0.laz') + pcloud0_0 = pcu.PointCloud.from_las_file(dirname / 'leg000_points_dev0.laz') + pcloud0_0.assert_equals(pcloud0_0_ref) + pcloud0_1_ref = pcu.PointCloud.from_las_file(regression_data / 'als_multichannel_split_leg000_points_dev1.laz') + pcloud0_1 = pcu.PointCloud.from_las_file(dirname / 'leg000_points_dev1.laz') + pcloud0_1.assert_equals(pcloud0_1_ref) + pcloud0_2_ref = pcu.PointCloud.from_las_file(regression_data / 'als_multichannel_split_leg000_points_dev2.laz') + pcloud0_2 = pcu.PointCloud.from_las_file(dirname / 'leg000_points_dev2.laz') + pcloud0_2.assert_equals(pcloud0_2_ref) + pcloud1_0_ref = pcu.PointCloud.from_las_file(regression_data / 'als_multichannel_split_leg002_points_dev0.laz') + pcloud1_0 = pcu.PointCloud.from_las_file(dirname / 'leg002_points_dev0.laz') + pcloud1_0.assert_equals(pcloud1_0_ref) + pcloud1_1_ref = pcu.PointCloud.from_las_file(regression_data / 'als_multichannel_split_leg002_points_dev1.laz') + pcloud1_1 = pcu.PointCloud.from_las_file(dirname / 'leg002_points_dev1.laz') + pcloud1_1.assert_equals(pcloud1_1_ref) + pcloud1_2_ref = pcu.PointCloud.from_las_file(regression_data / 'als_multichannel_split_leg002_points_dev2.laz') + pcloud1_2 = pcu.PointCloud.from_las_file(dirname / 'leg002_points_dev2.laz') + pcloud1_2.assert_equals(pcloud1_2_ref) @pytest.mark.skipif("laspy" not in sys.modules, @@ -358,10 +427,10 @@ def eval_als_multichannel_split(dirname): pytest.param(True, True, id="LAZ v1.0"), ] ) -def test_las_pyh(zip_flag: bool, las10_flag: bool): +def test_las_pyh(zip_flag: bool, las10_flag: bool, output_dir): """""" dirname_pyh = run_helios_pyhelios(Path('data') / 'surveys' / 'demo' / 'light_als_toyblocks_multiscanner.xml', - las_output=True, zip_output=zip_flag, las10=las10_flag) + output=output_dir, las_output=True, zip_output=zip_flag, las10=las10_flag) las_version = "1.0" if las10_flag else "1.4" eval_las(dirname_pyh, las_version) @@ -378,7 +447,7 @@ def test_las_pyh(zip_flag: bool, las10_flag: bool): pytest.param(True, True, id="LAZ v1.0"), ] ) -def test_las_exe(zip_flag: bool, las10_flag: bool): +def test_las_exe(zip_flag: bool, las10_flag: bool, output_dir): options = ["--lasOutput"] if zip_flag: options.append("--zipOutput") @@ -386,7 +455,9 @@ def test_las_exe(zip_flag: bool, las10_flag: bool): options.append("--las10") dirname_exe = run_helios_executable(Path('data') / 'surveys' / 'demo' / 'light_als_toyblocks_multiscanner.xml', - options=options) + output_dir, + options=options, + ) las_version = "1.0" if las10_flag else "1.4" eval_las(dirname_exe, las_version) @@ -415,10 +486,10 @@ def eval_las(dirname, las_version, check_empty=False): @pytest.mark.skipif("laspy" not in sys.modules, reason="requires the laspy library") @pytest.mark.pyh -def test_strip_id_pyh(): +def test_strip_id_pyh(output_dir): """""" dirname_pyh = run_helios_pyhelios(Path('data') / 'test' / 'als_hd_height_above_ground_stripid_light.xml', - las_output=True, zip_output=True) + output=output_dir, las_output=True, zip_output=True) las_version = "1.4" eval_las(dirname_pyh, las_version, check_empty=False) @@ -426,31 +497,35 @@ def test_strip_id_pyh(): @pytest.mark.skipif("laspy" not in sys.modules, reason="requires the laspy library") @pytest.mark.exe -def test_strip_id_exe(): +def test_strip_id_exe(output_dir): dirname_exe = run_helios_executable(Path('data') / 'test' / 'als_hd_height_above_ground_stripid_light.xml', + output_dir, options=['--lasOutput', '--zipOutput']) las_version = "1.4" eval_las(dirname_exe, las_version, check_empty=True) @pytest.mark.pyh -def test_dyn_pyh(): +def test_dyn_pyh(regression_data, output_dir): dirname_pyh = run_helios_pyhelios(Path('data') / 'surveys' / 'dyn' / 'tls_dyn_cube.xml', - las_output=True, zip_output=True, + output=output_dir, las_output=True, zip_output=True, start_time='2022-01-01 00:00:00') - eval_dyn(dirname_pyh) + eval_dyn(regression_data, dirname_pyh) @pytest.mark.exe -def test_dyn_exe(): +def test_dyn_exe(regression_data, output_dir): dirname_exe = run_helios_executable(Path('data') / 'surveys' / 'dyn' / 'tls_dyn_cube.xml', + output_dir, options=['--lasOutput', '--zipOutput', - '--gpsStartTime', '2022-01-01 00:00:00']) - eval_dyn(dirname_exe) + '--gpsStartTime', '2022-01-01 00:00:00' + ]) + eval_dyn(regression_data, dirname_exe) -def eval_dyn(dirname): +def eval_dyn(regression_data, dirname): assert (dirname / 'leg000_points.laz').exists() - assert abs((dirname / 'leg000_points.laz').stat().st_size - 2_642_291) < MAX_DIFFERENCE_BYTES - # clean up - if DELETE_FILES_AFTER: - shutil.rmtree(dirname) + + if regression_data: + pcloud0_ref = pcu.PointCloud.from_las_file(regression_data / 'dyn_leg000_points.laz') + pcloud0 = pcu.PointCloud.from_las_file(dirname / 'leg000_points.laz') + pcloud0.assert_equals(pcloud0_ref) diff --git a/pytests/test_gpsStartTimeFlag.py b/pytests/test_gpsStartTimeFlag.py index 7e1291167..5b7cd2be1 100644 --- a/pytests/test_gpsStartTimeFlag.py +++ b/pytests/test_gpsStartTimeFlag.py @@ -1,35 +1,17 @@ -import os -import shutil -from test_demo_scenes import run_helios_executable, find_playback_dir +from .test_demo_scenes import run_helios_executable, find_playback_dir from pathlib import Path -import sys import datetime -import hashlib -MAX_DIFFERENCE_BYTES = 1024 -DELETE_FILES_AFTER = True -WORKING_DIR = os.getcwd() -def sha256sum(filename): - h = hashlib.sha256() - b = bytearray(128*1024) - mv = memoryview(b) - with open(filename, 'rb', buffering=0) as f: - for n in iter(lambda : f.readinto(mv), 0): - h.update(mv[:n]) - return h.hexdigest() - - -def run_helios_pyhelios(survey_path: Path, options=None) -> Path: - sys.path.append(WORKING_DIR) +def run_helios_pyhelios(survey_path: Path, output_dir: Path, options=None) -> Path: import pyhelios pyhelios.setDefaultRandomnessGeneratorSeed("43") from pyhelios import SimulationBuilder simB = SimulationBuilder( surveyPath=str(survey_path.absolute()), - assetsDir=WORKING_DIR + os.sep + 'assets' + os.sep, - outputDir=WORKING_DIR + os.sep + 'output' + os.sep, + assetsDir=[str(Path("assets"))], + outputDir=str(output_dir), ) simB.setLasOutput(False) simB.setRebuildScene(True) @@ -41,64 +23,44 @@ def run_helios_pyhelios(survey_path: Path, options=None) -> Path: sim.start() output = sim.join() sim = None - return find_playback_dir(survey_path) + return find_playback_dir(survey_path, output_dir) + -def test_gpsStartTimeFlag_exe(): +def test_gpsStartTimeFlag_exe(output_dir): now = datetime.datetime(year=1994, month=1, day=18, hour=1, minute=45, second=22) unixtime = datetime.datetime.timestamp(now) stringtime = now.strftime('%Y-%m-%d %H:%M:%S') # first, run 'as is': - r1 = run_helios_executable(Path(WORKING_DIR) / 'data' / 'surveys' / 'demo' / 'tls_arbaro_demo.xml', - options=['--gpsStartTime', '']) + r1 = run_helios_executable(Path('data') / 'surveys' / 'demo' / 'tls_arbaro_demo.xml', + output_dir, + options=['--gpsStartTime', '']) # run with posix ts: - r2 = run_helios_executable(Path(WORKING_DIR) / 'data' / 'surveys' / 'demo' / 'tls_arbaro_demo.xml', - options=['--gpsStartTime', f'{unixtime:.3f}']) + r2 = run_helios_executable(Path('data') / 'surveys' / 'demo' / 'tls_arbaro_demo.xml', + output_dir, + options=['--gpsStartTime', f'{unixtime:.3f}']) # run with string ts: - r3 = run_helios_executable(Path(WORKING_DIR) / 'data' / 'surveys' / 'demo' / 'tls_arbaro_demo.xml', - options=['--gpsStartTime', stringtime]) + r3 = run_helios_executable(Path('data') / 'surveys' / 'demo' / 'tls_arbaro_demo.xml', + output_dir, + options=['--gpsStartTime', stringtime]) assert (r1 / 'leg000_points.xyz').exists() - r1_sum = sha256sum(r1 / 'leg000_points.xyz') - r2_sum = sha256sum(r2 / 'leg000_points.xyz') - r3_sum = sha256sum(r3 / 'leg000_points.xyz') - # assert r2_sum == r3_sum - # assert r2_sum == '41313dfe46ed34fcb9733af03a4d5e52487fd4579014f13dc00c609b53813229' or \ - # r2_sum == '984cfbbc5a54ab10a566ea901363218f35da569dbab5cd102424ab27794074ae' # linux checksum - # assert r1_sum != r2_sum - if DELETE_FILES_AFTER: - shutil.rmtree(r1) - shutil.rmtree(r2) - shutil.rmtree(r3) - -def test_gpsStartTimeFlag_pyh(): +def test_gpsStartTimeFlag_pyh(output_dir): now = datetime.datetime(year=1994, month=1, day=18, hour=1, minute=45, second=22) unixtime = datetime.datetime.timestamp(now) stringtime = now.strftime('%Y-%m-%d %H:%M:%S') # first, run 'as is': - r1 = run_helios_pyhelios(Path(WORKING_DIR) / 'data' / 'surveys' / 'demo' / 'tls_arbaro_demo.xml', - options={'gpsStartTime': ''}) + r1 = run_helios_pyhelios(Path('data') / 'surveys' / 'demo' / 'tls_arbaro_demo.xml', + output_dir, + options={'gpsStartTime': ''}) # run with posix ts: - r2 = run_helios_pyhelios(Path(WORKING_DIR) / 'data' / 'surveys' / 'demo' / 'tls_arbaro_demo.xml', - options={'gpsStartTime': f'{unixtime:.0f}'}) + r2 = run_helios_pyhelios(Path('data') / 'surveys' / 'demo' / 'tls_arbaro_demo.xml', + output_dir, + options={'gpsStartTime': f'{unixtime:.0f}'}) # run with string ts: - r3 = run_helios_pyhelios(Path(WORKING_DIR) / 'data' / 'surveys' / 'demo' / 'tls_arbaro_demo.xml', - options={'gpsStartTime': stringtime}) + r3 = run_helios_pyhelios(Path('data') / 'surveys' / 'demo' / 'tls_arbaro_demo.xml', + output_dir, + options={'gpsStartTime': stringtime}) assert (r1 / 'leg000_points.xyz').exists() - r1_sum = sha256sum(r1 / 'leg000_points.xyz') - r2_sum = sha256sum(r2 / 'leg000_points.xyz') - r3_sum = sha256sum(r3 / 'leg000_points.xyz') - # assert r2_sum == r3_sum - # assert r2_sum == '41313dfe46ed34fcb9733af03a4d5e52487fd4579014f13dc00c609b53813229' or \ - # r2_sum == '984cfbbc5a54ab10a566ea901363218f35da569dbab5cd102424ab27794074ae' # linux checksum - # assert r1_sum != r2_sum - - if DELETE_FILES_AFTER: - try: - shutil.rmtree(r1) - shutil.rmtree(r2) - shutil.rmtree(r3) - except Exception as e: - print(f"Error cleaning up: {e}") diff --git a/pytests/test_pyhelios.py b/pytests/test_pyhelios.py index 9a775fee2..5e8c61c40 100644 --- a/pytests/test_pyhelios.py +++ b/pytests/test_pyhelios.py @@ -12,12 +12,11 @@ import time import struct import xml.etree.ElementTree as ET -import shutil +import laspy - -DELETE_FILES_AFTER = False -WORKING_DIR = os.getcwd() import pyhelios +from .test_demo_scenes import find_playback_dir +from . import pcloud_utils as pcu def find_scene(survey_file): @@ -46,7 +45,7 @@ def get_las_version(las_filename): @pytest.fixture(scope="session") -def test_sim(): +def test_sim(output_dir): """ Fixture which returns a simulation object for a given survey path """ @@ -56,8 +55,8 @@ def create_test_sim(survey_path, zip_output=True, las_output=True, las10=False): from pyhelios import SimulationBuilder simB = SimulationBuilder( surveyPath=str(survey_path.absolute()), - assetsDir=WORKING_DIR + os.sep + 'assets' + os.sep, - outputDir=WORKING_DIR + os.sep + 'output' + os.sep, + assetsDir=[str(Path('assets'))], + outputDir=str(output_dir), ) simB.setLasOutput(las_output) simB.setLas10(las10) @@ -184,7 +183,6 @@ def test_survey_characteristics(test_sim): """Test accessing survey characteristics (name, length)""" path_to_survey = Path('data') / 'surveys' / 'toyblocks' / 'als_toyblocks.xml' sim = test_sim(path_to_survey) - assert Path(sim.sim.getSurveyPath()) == Path(WORKING_DIR) / path_to_survey survey = sim.sim.getSurvey() assert survey.name == 'toyblocks_als' assert survey.getLength() == 0.0 @@ -196,7 +194,7 @@ def test_scene(): pass -def test_create_survey(): +def test_create_survey(output_dir): """Test creating/configuring a survey with pyhelios""" pyhelios.setDefaultRandomnessGeneratorSeed("7") test_survey_path = 'data/surveys/test_survey.xml' @@ -216,7 +214,7 @@ def test_create_survey(): simBuilder = pyhelios.SimulationBuilder( test_survey_path, 'assets/', - 'output/' + str(output_dir) ) simBuilder.setFinalOutput(True) simBuilder.setLasOutput(True) @@ -273,18 +271,12 @@ def test_create_survey(): output = simB.join() meas, traj = pyhelios.outputToNumpy(output) # check length of output - assert meas.shape == (9926, 17) + assert meas.shape == (10531, 17) assert traj.shape == (6670, 7) # compare individual points np.testing.assert_allclose(meas[100, :3], np.array([83.32, -66.44204, 0.03114649])) np.testing.assert_allclose(traj[0, :3], np.array([waypoints[0][0], waypoints[0][1], altitude])) - # cleanup - os.remove(test_survey_path) - if DELETE_FILES_AFTER: - print(f"Deleting files in {Path(output.outpath).parent.as_posix()}") - shutil.rmtree(Path(output.outpath).parent) - def test_material(test_sim): """Test accessing material properties of a primitive in a scene""" @@ -344,15 +336,15 @@ def test_detector(test_sim): [pytest.param(True, id="setExportToFile(True)"), pytest.param(False, id="setExportToFile(False)")] ) -def test_output(export_to_file): +def test_output(output_dir, export_to_file): """Validating the output of a survey started with pyhelios""" from pyhelios import SimulationBuilder survey_path = Path('data') / 'test' / 'als_hd_demo_tiff_min.xml' pyhelios.setDefaultRandomnessGeneratorSeed("43") simB = SimulationBuilder( surveyPath=str(survey_path.absolute()), - assetsDir=WORKING_DIR + os.sep + 'assets' + os.sep, - outputDir=WORKING_DIR + os.sep + 'output' + os.sep, + assetsDir=[str(Path('assets'))], + outputDir=str(output_dir), ) simB.setFinalOutput(True) simB.setExportToFile(export_to_file) @@ -360,6 +352,7 @@ def test_output(export_to_file): simB.setCallbackFrequency(100) simB.setNumThreads(1) simB.setKDTJobs(1) + simB.setFixedGpsTimeStart('2022-01-01 00:00:00') sim = simB.build() @@ -367,12 +360,16 @@ def test_output(export_to_file): output = sim.join() measurements_array, trajectory_array = pyhelios.outputToNumpy(output) - np.testing.assert_allclose(measurements_array[0, :3], np.array([474500.3, 5473580.0, 107.0001]), rtol=0.000001) - assert measurements_array.shape == (2407, 17) - assert trajectory_array.shape == (9, 7) if export_to_file: - assert Path(output.outpath).parent.parent == Path(WORKING_DIR) / "output" / "als_hd_demo" - # cleanup - if DELETE_FILES_AFTER: - print(f"Deleting files in {Path(output.outpath).parent.as_posix()}") - shutil.rmtree(Path(output.outpath).parent) + # check if output files exist + dirname = find_playback_dir(survey_path, output_dir) + assert (Path(dirname) / 'leg000_points.xyz').exists() + assert (Path(dirname) / 'leg001_points.xyz').exists() + assert (Path(dirname) / 'leg002_points.xyz').exists() + # check if output files contain the same data as the internal arrays + leg0 = np.loadtxt(Path(dirname) / 'leg000_points.xyz') + leg1 = np.loadtxt(Path(dirname) / 'leg001_points.xyz') + leg2 = np.loadtxt(Path(dirname) / 'leg002_points.xyz') + measurements_from_file = np.vstack((leg0, leg1, leg2)) + # account for different (order of) fields in internal array and file + assert np.allclose(measurements_array[:, (0, 1, 2, 9, 10, 12, 11, 13, 14, 15, 16)], measurements_from_file, atol=1e-6) diff --git a/python/pyhelios/simulation_build.py b/python/pyhelios/simulation_build.py index a5db4ca4f..b777fb7cc 100644 --- a/python/pyhelios/simulation_build.py +++ b/python/pyhelios/simulation_build.py @@ -85,7 +85,9 @@ def copy(self): Return: SimulationBuild which is a copy of current one """ - copySim = SimulationBuild(None, None, None, None, None, None, True) + copySim = SimulationBuild( + None, None, None, None, None, None, None, None, True + ) copySim.sim = self.sim.copy() return copySim diff --git a/python/pyhelios/simulation_builder.py b/python/pyhelios/simulation_builder.py index b4c456d14..c830c8d60 100644 --- a/python/pyhelios/simulation_builder.py +++ b/python/pyhelios/simulation_builder.py @@ -103,7 +103,7 @@ def makeDefault(self): self.setCalcEchowidth(False) self.setFullwaveNoise(False) self.setPlatformNoiseDisabled(True) - self.setLegacyEnergyModel(False) + self.setLegacyEnergyModel(True) self.setExportToFile(True) self.setCallback(None) self.rotateFilters = [] @@ -156,7 +156,8 @@ def build(self): self.writeWaveform, self.calcEchowidth, self.fullwaveNoise, - self.platformNoiseDisabled + self.platformNoiseDisabled, + False ) if self.callback is not None: build.sim.setCallback(self.callback) diff --git a/src/filems/read/core/DesignMatrixReader.tpp b/src/filems/read/core/DesignMatrixReader.tpp index 180ac0a29..f20e0a935 100644 --- a/src/filems/read/core/DesignMatrixReader.tpp +++ b/src/filems/read/core/DesignMatrixReader.tpp @@ -23,7 +23,7 @@ DesignMatrix DesignMatrixReader::read( bool firstRow = true; size_t nValuesPerRow = 0; try{ - // Parsing loop : fill varialbes + // Parsing loop : fill variables while(true){ string const str = br.read(); size_t const comIdx = str.find(com); diff --git a/src/main/helios_version.cpp b/src/main/helios_version.cpp index 5ac93ee31..0242f54a3 100644 --- a/src/main/helios_version.cpp +++ b/src/main/helios_version.cpp @@ -4,7 +4,7 @@ const char * HELIOS_VERSION = "1.3.0"; -const char * HELIOS_GIT_HASH = "0dd016f8"; +const char * HELIOS_GIT_HASH = "917263d4"; const char * getHeliosVersion(){ return HELIOS_VERSION; diff --git a/src/maths/fluxionum/AbstractDesignMatrix.h b/src/maths/fluxionum/AbstractDesignMatrix.h index e8529938b..c7bc71e22 100644 --- a/src/maths/fluxionum/AbstractDesignMatrix.h +++ b/src/maths/fluxionum/AbstractDesignMatrix.h @@ -50,7 +50,6 @@ class AbstractDesignMatrix{ virtual ~AbstractDesignMatrix() = default; -protected: // *** INNER UTILS *** // // ********************* // /** @@ -60,8 +59,8 @@ class AbstractDesignMatrix{ * then a fluxionum::FluxionumException will be thrown */ inline size_t translateColumnNameToIndex(string const &columnName) const{ - for(string const &name : columnNames){ - if(name == columnName) return true; + for(size_t i = 0 ; i < columnNames.size() ; ++i){ + if(columnNames[i] == columnName) return i; } std::stringstream ss; ss << "AbstractDesignMatrix::translateColumnNameToIndex failed to " diff --git a/src/maths/fluxionum/DesignMatrix.h b/src/maths/fluxionum/DesignMatrix.h index e4408f5b4..6697c82ad 100644 --- a/src/maths/fluxionum/DesignMatrix.h +++ b/src/maths/fluxionum/DesignMatrix.h @@ -39,11 +39,12 @@ using std::string; */ template class DesignMatrix : public AbstractDesignMatrix{ -protected: +public: // Originally it was protected, it has been changed to public // *** USING *** // // *************** // using AbstractDesignMatrix::translateColumnNameToIndex; +protected: // *** ATTRIBUTES *** // // ******************** // /** diff --git a/src/maths/fluxionum/TemporalDesignMatrix.h b/src/maths/fluxionum/TemporalDesignMatrix.h index 65f6e04fd..1853f7496 100644 --- a/src/maths/fluxionum/TemporalDesignMatrix.h +++ b/src/maths/fluxionum/TemporalDesignMatrix.h @@ -249,9 +249,11 @@ class TemporalDesignMatrix : public DesignMatrix { helios::filems::DesignMatrixReader reader(path, sep); std::unordered_map kv; DesignMatrix const dm = reader.read(&kv); - size_t const tCol = (size_t) std::strtoul( - kv.at("TIME_COLUMN").c_str(), nullptr, 10 - ); + size_t const tCol = (kv.find("TIME_COLUMN") == kv.end()) ? + dm.translateColumnNameToIndex("t") : + (size_t) std::strtoul( + kv.at("TIME_COLUMN").c_str(), nullptr, 10 + ); *this = TemporalDesignMatrix( dm, tCol, diff --git a/src/pybinds/PyHelios.cpp b/src/pybinds/PyHelios.cpp index 5d1ecebfb..ded19619b 100644 --- a/src/pybinds/PyHelios.cpp +++ b/src/pybinds/PyHelios.cpp @@ -155,6 +155,11 @@ BOOST_PYTHON_MODULE(_pyhelios){ &PyHeliosSimulation::newLeg, return_internal_reference<>() ) + .def( + "newLegFromTemplate", + &PyHeliosSimulation::newLegFromTemplate, + return_internal_reference<>() + ) .def( "newScanningStrip", &PyHeliosSimulation::newScanningStrip, diff --git a/src/pybinds/PyHeliosSimulation.cpp b/src/pybinds/PyHeliosSimulation.cpp index 7d420f1fd..a245e472b 100644 --- a/src/pybinds/PyHeliosSimulation.cpp +++ b/src/pybinds/PyHeliosSimulation.cpp @@ -104,10 +104,20 @@ Leg & PyHeliosSimulation::newLeg(int index){ int const n = (int) survey->legs.size(); if(index<0 || index>n) index = n; std::shared_ptr leg = std::make_shared(); - leg->mScannerSettings = - std::make_shared(); - leg->mPlatformSettings = - std::make_shared(); + leg->mScannerSettings = std::make_shared(); + leg->mPlatformSettings = std::make_shared(); + survey->addLeg(index, leg); + return *leg; +} + +Leg & PyHeliosSimulation::newLegFromTemplate( + int index, + Leg &baseLeg +){ + int const n = (int) survey->legs.size(); + if(index<0 || index>n) index = n; + std::shared_ptr leg = std::make_shared(baseLeg); + leg->setSerialId(index); survey->addLeg(index, leg); return *leg; } @@ -318,13 +328,15 @@ void PyHeliosSimulation::loadSurvey( bool writeWaveform, bool calcEchowidth, bool fullWaveNoise, - bool platformNoiseDisabled + bool platformNoiseDisabled, + bool writePulse ){ xmlreader->sceneLoader.kdtFactoryType = kdtFactory; xmlreader->sceneLoader.kdtNumJobs = kdtJobs; xmlreader->sceneLoader.kdtSAHLossNodes = kdtSAHLossNodes; survey = xmlreader->load(legNoiseDisabled, rebuildScene); survey->scanner->setWriteWaveform(writeWaveform); + survey->scanner->setWritePulse(writePulse); survey->scanner->setCalcEchowidth(calcEchowidth); survey->scanner->setFullWaveNoise(fullWaveNoise); survey->scanner->setPlatformNoiseDisabled(platformNoiseDisabled); diff --git a/src/pybinds/PyHeliosSimulation.h b/src/pybinds/PyHeliosSimulation.h index 61a5e9d67..94b2f265b 100644 --- a/src/pybinds/PyHeliosSimulation.h +++ b/src/pybinds/PyHeliosSimulation.h @@ -177,11 +177,23 @@ class PyHeliosSimulation{ {survey->legs.erase(survey->legs.begin() + index);} /** * @brief Create a new empty leg - * @param index The index specifying the position in the survey where the - * leg will be inserted - * @return Created empty leg + * @param index The index specifying the position in the survey where the + * leg will be inserted. + * @return Created leg. */ Leg & newLeg(int index); + /** + * @brief Create a new leg from a template. + * @param index The index specifying the position in the survey where the + * leg will be inserted. + * @param baseLeg The leg to be used as a template to build the new leg. + * If null, the new leg will be created fully from scratch. + * @return Created leg. + */ + Leg & newLegFromTemplate( + int index, + Leg &baseLeg + ); /** * @brief Create a new empty scanning strip (with no legs) * @param stripId The identifier for the strip @@ -391,7 +403,8 @@ class PyHeliosSimulation{ bool writeWaveform = false, bool calcEchowidth = false, bool fullWaveNoise = false, - bool platformNoiseDisabled = true + bool platformNoiseDisabled = true, + bool writePulse = false ); void addRotateFilter( double q0, diff --git a/src/pybinds/PyMeasurementWrapper.h b/src/pybinds/PyMeasurementWrapper.h index e58b87e84..ad81ed870 100644 --- a/src/pybinds/PyMeasurementWrapper.h +++ b/src/pybinds/PyMeasurementWrapper.h @@ -54,8 +54,8 @@ class PyMeasurementWrapper{ int getClassification() {return m.classification;} void setClassification(int classification) {m.classification = classification;} - long getGpsTime() {return m.gpsTime;} - void setGpsTime(long gpsTime) {m.gpsTime = gpsTime;} + double getGpsTime() {return F64((m.gpsTime)/1000000000.0);} + void setGpsTime(double gpsTime) {m.gpsTime = gpsTime * 1000000000.0;} }; } diff --git a/src/scanner/MultiScanner.cpp b/src/scanner/MultiScanner.cpp index 860d2da21..7e610b657 100644 --- a/src/scanner/MultiScanner.cpp +++ b/src/scanner/MultiScanner.cpp @@ -8,7 +8,7 @@ // ************************************ // MultiScanner::MultiScanner(MultiScanner &scanner) : Scanner(scanner), - scanDevs(std::move(scanner.scanDevs)) + scanDevs(scanner.scanDevs) {} diff --git a/src/sim/comps/Survey.cpp b/src/sim/comps/Survey.cpp index 11cb76ec8..41c003de5 100644 --- a/src/sim/comps/Survey.cpp +++ b/src/sim/comps/Survey.cpp @@ -19,7 +19,9 @@ Survey::Survey(Survey &survey, bool const deepCopy){ // Copy Scanner this->scanner = survey.scanner->clone(); - this->scanner->getDetector()->scanner = this->scanner; + for(size_t i = 0 ; i < this->scanner->getNumDevices() ; ++i){ + this->scanner->getDetector(i)->scanner = this->scanner; + } // Copy legs this->legs = std::vector>(0); diff --git a/src/sim/core/SurveyPlayback.cpp b/src/sim/core/SurveyPlayback.cpp index 838e5da21..a382a01fc 100644 --- a/src/sim/core/SurveyPlayback.cpp +++ b/src/sim/core/SurveyPlayback.cpp @@ -517,7 +517,7 @@ void SurveyPlayback::clearPointcloudFile(){ // Dont clear strip file, it would overwrite previous point cloud content if(getCurrentLeg()->isContainedInAStrip()) return; // Dont clear pcloud file, if leg is not active there is nothing to clear - // Otherwise, WATCHOUT because last active leg might be overwriten + // Otherwise, WATCH OUT because last active leg might be overwriten if(!getCurrentLeg()->mScannerSettings->active) return; fms->write.clearPointcloudFile(); }