Skip to content
Build

add support for OpenSearch provider #3140

Sign in to view logs

add support for OpenSearch provider

add support for OpenSearch provider #3140

Workflow file for this run

.github/workflows/main.yml at 4a29d3d
name: Build
on:
push:
paths-ignore:
- '**.md'
- 'docs/**'
pull_request:
branches:
- master
paths-ignore:
- '!**.md'
- 'docs/**'
release:
types:
- released
jobs:
main:
runs-on: ubuntu-22.04
strategy:
matrix:
include:
- python-version: '3.10'
env:
PYGEOAPI_CONFIG: "$(pwd)/pygeoapi-config.yml"
steps:
- name: Clear up GitHub runner diskspace
run: |
echo "Space before"
df -h /
sudo rm -rf /usr/local/lib/android
sudo rm -rf /usr/share/dotnet
echo "Space after"
df -h /
- name: Chown user
run: |
sudo chown -R $USER:$USER $GITHUB_WORKSPACE
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
name: Setup Python ${{ matrix.python-version }}
with:
python-version: ${{ matrix.python-version }}
- name: Configure sysctl limits
run: |
sudo swapoff -a
sudo sysctl -w vm.swappiness=1
sudo sysctl -w fs.file-max=262144
sudo sysctl -w vm.max_map_count=262144
- name: Install and run PostgreSQL/PostGIS 📦
uses: huaxk/postgis-action@v1
with:
postgresql password: ${{ secrets.DatabasePassword || 'postgres' }}
postgresql db: 'test'
- name: Install and run Elasticsearch 📦
uses: getong/[email protected]
with:
elasticsearch version: '8.3.1'
host port: 9200
container port: 9200
host node port: 9300
node port: 9300
discovery type: 'single-node'
- name: Install and run OpenSearch 📦
uses: esmarkowski/[email protected]
with:
version: 2.12.0
security-disabled: true
port: 9209
- name: Install and run MongoDB
uses: supercharge/[email protected]
with:
mongodb-version: 4.4
- name: Install and run SensorThingsAPI
uses: cgs-earth/[email protected]
- name: Install sqlite and gpkg dependencies
uses: awalsh128/cache-apt-pkgs-action@latest
with:
packages: libsqlite3-mod-spatialite
version: 4.3.0a-6build1
- name: Use ubuntuGIS unstable ppa
run: sudo add-apt-repository ppa:ubuntugis/ubuntugis-unstable && sudo apt update
shell: bash
- name: Install GDAL with Python bindings
uses: awalsh128/cache-apt-pkgs-action@latest
with:
packages: gdal-bin libgdal-dev
version: 3.8.4
- name: Install and run Oracle
run: |
docker run -d --name oracledb -e ORACLE_PWD=oracle -v ${{ github.workspace }}/tests/data/oracle/init-db:/opt/oracle/scripts/startup -p 1521:1521 container-registry.oracle.com/database/express:21.3.0-xe
- name: Install requirements 📦
run: |
pip3 install -r requirements.txt
pip3 install -r requirements-admin.txt
pip3 install -r requirements-starlette.txt
pip3 install -r requirements-dev.txt
pip3 install -r requirements-provider.txt
pip3 install -r requirements-manager.txt
pip3 install -r requirements-django.txt
python3 setup.py install
pip3 install --global-option=build_ext --global-option="-I/usr/include/gdal" GDAL==`gdal-config --version`
#pip3 install --upgrade rasterio==1.1.8
- name: setup test data ⚙️
run: |
python3 tests/load_es_data.py tests/data/ne_110m_populated_places_simple.geojson geonameid
python3 tests/load_opensearch_data.py tests/data/ne_110m_populated_places_simple.geojson geonameid
python3 tests/load_mongo_data.py tests/data/ne_110m_populated_places_simple.geojson
gunzip < tests/data/hotosm_bdi_waterways.sql.gz | psql postgresql://postgres:${{ secrets.DatabasePassword || 'postgres' }}@localhost:5432/test
psql postgresql://postgres:${{ secrets.DatabasePassword || 'postgres' }}@localhost:5432/test -f tests/data/dummy_data.sql
psql postgresql://postgres:${{ secrets.DatabasePassword || 'postgres' }}@localhost:5432/test -f tests/data/dummy_types_data.sql
psql postgresql://postgres:${{ secrets.DatabasePassword || 'postgres' }}@localhost:5432/test -f tests/data/postgres_manager_full_structure.backup.sql
docker ps
python3 tests/load_oracle_data.py
- name: run unit tests ⚙️
env:
POSTGRESQL_PASSWORD: ${{ secrets.DatabasePassword || 'postgres' }}
run: |
pytest tests/api
pytest tests/test_api_ogr_provider.py
pytest tests/test_config.py
pytest tests/test_csv__formatter.py
pytest tests/test_csv__provider.py
pytest tests/test_django.py
pytest tests/test_elasticsearch__provider.py
pytest tests/test_opensearch__provider.py
pytest tests/test_esri_provider.py
pytest tests/test_filesystem_provider.py
pytest tests/test_geojson_provider.py
pytest tests/test_mongo_provider.py
pytest tests/test_ogr_csv_provider.py
pytest tests/test_ogr_esrijson_provider.py
pytest tests/test_ogr_gpkg_provider.py
pytest tests/test_ogr_shapefile_provider.py
pytest tests/test_ogr_sqlite_provider.py
pytest tests/test_ogr_wfs_provider.py
pytest tests/test_postgresql_manager.py
# pytest tests/test_ogr_wfs_provider_live.py # NOTE: these are skipped in the file but listed here for completeness
pytest tests/test_openapi.py
pytest tests/test_oracle_provider.py
pytest tests/test_postgresql_provider.py
pytest tests/test_rasterio_provider.py
pytest tests/test_sensorthings_provider.py
pytest tests/test_socrata_provider.py
pytest tests/test_sqlite_geopackage_provider.py
pytest tests/test_tinydb_catalogue_provider.py
pytest tests/test_tinydb_manager_for_parallel_requests.py
pytest tests/test_util.py
pytest tests/test_xarray_netcdf_provider.py
pytest tests/test_xarray_zarr_provider.py
- name: failed tests 🚩
if: ${{ failure() }}
run: |
pip3 list -v
admin:
runs-on: ubuntu-22.04
strategy:
matrix:
include:
- python-version: '3.10'
env:
PYGEOAPI_CONFIG: "tests/pygeoapi-test-config-admin.yml"
PYGEOAPI_OPENAPI: "tests/pygeoapi-test-openapi-admin.yml"
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
name: Setup Python ${{ matrix.python-version }}
with:
python-version: ${{ matrix.python-version }}
- uses: awalsh128/cache-apt-pkgs-action@latest
with:
packages: gunicorn python3-gevent
version: 1.0
- name: Install requirements 📦
run: |
pip3 install -r requirements.txt
pip3 install -r requirements-dev.txt
pip3 install -r requirements-admin.txt
python3 setup.py install
- name: Run pygeoapi with admin API ⚙️
run: |
pygeoapi openapi generate ${PYGEOAPI_CONFIG} --output-file ${PYGEOAPI_OPENAPI}
gunicorn --bind 0.0.0.0:5000 \
--reload \
--reload-extra-file ${PYGEOAPI_CONFIG} \
pygeoapi.flask_app:APP &
- name: run integration tests ⚙️
run: |
pytest tests/test_admin_api.py
- name: failed tests 🚩
if: ${{ failure() }}
run: |
pip3 list -v