Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

nix: Make postgrest-with-postgresql-xxx postgrest-test-io work better #3221

Merged
merged 1 commit into from
Feb 10, 2024
Merged

nix: Make postgrest-with-postgresql-xxx postgrest-test-io work better #3221

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ jobs:

- name: Run IO tests
if: always()
run: postgrest-with-postgresql-${{ matrix.pgVersion }} -f test/io/fixtures.sql postgrest-test-io -vv
run: postgrest-with-postgresql-${{ matrix.pgVersion }} postgrest-test-io -vv


Test-Memory-Nix:
Expand Down
2 changes: 1 addition & 1 deletion nix/tools/loadtest.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ let
abs_output="$(realpath "$_arg_output")"

# shellcheck disable=SC2145
${withTools.withPg} --fixtures "$_arg_testdir"/fixtures.sql \
${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
${withTools.withSlowPg} \
${withTools.withPgrst} \
${withTools.withSlowPgrst} \
Expand Down
2 changes: 1 addition & 1 deletion nix/tools/memory.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ let
withPath = [ postgrestProfiled curl ];
}
''
${withTools.withPg} test/memory/memory-tests.sh
${withTools.withPg} -f test/spec/fixtures/load.sql test/memory/memory-tests.sh
'';

in
Expand Down
20 changes: 11 additions & 9 deletions nix/tools/tests.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ let
withEnv = postgrest.env;
}
''
${withTools.withPg} ${cabal-install}/bin/cabal v2-run ${devCabalOptions} \
test:spec -- "''${_arg_leftovers[@]}"
${withTools.withPg} -f test/spec/fixtures/load.sql \
${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec -- "''${_arg_leftovers[@]}"
'';

testDoctests =
Expand Down Expand Up @@ -59,9 +59,10 @@ let
withEnv = postgrest.env;
}
''
${withTools.withPg} ${runtimeShell} -c " \
${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec && \
${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec"
${withTools.withPg} -f test/spec/fixtures/load.sql \
${runtimeShell} -c " \
${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec && \
${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec"
'';

ioTestPython =
Expand Down Expand Up @@ -99,7 +100,7 @@ let
withPath = [ jq ];
}
''
${withTools.withPg} \
${withTools.withPg} -f test/spec/fixtures.sql \
${cabal-install}/bin/cabal v2-run ${devCabalOptions} --verbose=0 -- \
postgrest --dump-schema \
| ${yq}/bin/yq -y .
Expand Down Expand Up @@ -137,11 +138,12 @@ let

# collect all tests
HPCTIXFILE="$tmpdir"/io.tix \
${withTools.withPg} -f test/io/fixtures.sql ${cabal-install}/bin/cabal v2-exec ${devCabalOptions} -- \
${ioTestPython}/bin/pytest -v test/io
${withTools.withPg} -f test/io/fixtures.sql \
${cabal-install}/bin/cabal v2-exec ${devCabalOptions} -- ${ioTestPython}/bin/pytest -v test/io

HPCTIXFILE="$tmpdir"/spec.tix \
${withTools.withPg} ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec
${withTools.withPg} -f test/spec/fixtures/load.sql \
${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec

# Note: No coverage for doctests, as doctests leverage GHCi and GHCi does not support hpc

Expand Down
102 changes: 51 additions & 51 deletions nix/tools/withTools.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,14 @@ let
{ name, postgresql }:
let
commandName = "postgrest-with-${name}";
superuserRole = "postgres";
in
checkedShellScript
{
name = commandName;
docs = "Run the given command in a temporary database with ${name}. If you wish to mutate the database, login with the '${superuserRole}' role.";
docs = "Run the given command in a temporary database with ${name}. If you wish to mutate the database, login with the postgres role.";
args =
[
"ARG_OPTIONAL_SINGLE([fixtures], [f], [SQL file to load fixtures from], [test/spec/fixtures/load.sql])"
"ARG_OPTIONAL_SINGLE([fixtures], [f], [SQL file to load fixtures from])"
"ARG_POSITIONAL_SINGLE([command], [Command to run])"
"ARG_LEFTOVERS([command arguments])"
"ARG_USE_ENV([PGUSER], [postgrest_test_authenticator], [Authenticator PG role])"
Expand All @@ -41,63 +40,64 @@ let
withTmpDir = true;
}
''
# avoid starting multiple layers of withTmpDb
if test -v PGHOST; then
exec "$_arg_command" "''${_arg_leftovers[@]}"
fi

setuplog="$tmpdir/setup.log"

log () {
echo "$1" >> "$setuplog"
}

mkdir -p "$tmpdir"/{db,socket}
# remove data dir, even if we keep tmpdir - no need to upload it to artifacts
trap 'rm -rf $tmpdir/db' EXIT

export PGDATA="$tmpdir/db"
export PGHOST="$tmpdir/socket"
export PGUSER
export PGDATABASE
export PGRST_DB_SCHEMAS
export PGTZ
export PGOPTIONS

HBA_FILE="$tmpdir/pg_hba.conf"
echo "local $PGDATABASE some_protected_user password" > "$HBA_FILE"
echo "local $PGDATABASE all trust" >> "$HBA_FILE"

log "Initializing database cluster..."
# We try to make the database cluster as independent as possible from the host
# by specifying the timezone, locale and encoding.
# initdb -U creates a superuser(man initdb)
TZ=$PGTZ initdb --no-locale --encoding=UTF8 --nosync -U "${superuserRole}" --auth=trust \
>> "$setuplog"

log "Starting the database cluster..."
# Instead of listening on a local port, we will listen on a unix domain socket.
pg_ctl -l "$tmpdir/db.log" -w start -o "-F -c listen_addresses=\"\" -c hba_file=$HBA_FILE -k $PGHOST -c log_statement=\"all\" " \
>> "$setuplog"

# shellcheck disable=SC2317
stop () {
log "Stopping the database cluster..."
pg_ctl stop -m i >> "$setuplog"
rm -rf "$tmpdir/db"
}
trap stop EXIT

log "Creating a minimally privileged $PGUSER connection role..."
createuser "$PGUSER" -U "${superuserRole}" --host="$tmpdir/socket" --no-createdb --no-inherit --no-superuser --no-createrole --no-replication --login
# Avoid starting multiple layers of withTmpDb, but make sure to have the last invocation
# load fixtures. Otherwise postgrest-with-postgresql-xx postgrest-test-io would not be possible.
if ! test -v PGHOST; then

mkdir -p "$tmpdir"/{db,socket}
# remove data dir, even if we keep tmpdir - no need to upload it to artifacts
trap 'rm -rf $tmpdir/db' EXIT

export PGDATA="$tmpdir/db"
export PGHOST="$tmpdir/socket"
export PGUSER
export PGDATABASE
export PGRST_DB_SCHEMAS
export PGTZ
export PGOPTIONS

HBA_FILE="$tmpdir/pg_hba.conf"
echo "local $PGDATABASE some_protected_user password" > "$HBA_FILE"
echo "local $PGDATABASE all trust" >> "$HBA_FILE"

log "Initializing database cluster..."
# We try to make the database cluster as independent as possible from the host
# by specifying the timezone, locale and encoding.
# initdb -U creates a superuser(man initdb)
TZ=$PGTZ initdb --no-locale --encoding=UTF8 --nosync -U postgres --auth=trust \
>> "$setuplog"

log "Starting the database cluster..."
# Instead of listening on a local port, we will listen on a unix domain socket.
pg_ctl -l "$tmpdir/db.log" -w start -o "-F -c listen_addresses=\"\" -c hba_file=$HBA_FILE -k $PGHOST -c log_statement=\"all\" " \
>> "$setuplog"

# shellcheck disable=SC2317
stop () {
log "Stopping the database cluster..."
pg_ctl stop -m i >> "$setuplog"
rm -rf "$tmpdir/db"
}
trap stop EXIT

log "Loading fixtures under the ${superuserRole} role..."
psql -U "${superuserRole}" -v PGUSER="$PGUSER" -v ON_ERROR_STOP=1 -f "$_arg_fixtures" >> "$setuplog"
log "Creating a minimally privileged $PGUSER connection role..."
createuser "$PGUSER" -U postgres --host="$tmpdir/socket" --no-createdb --no-inherit --no-superuser --no-createrole --no-replication --login

log "Done. Running command..."
echo "${commandName}: You can connect with: psql 'postgres:///$PGDATABASE?host=$tmpdir/socket' -U postgres"
echo "${commandName}: You can tail the logs with: tail -f $tmpdir/db.log"
fi

echo "${commandName}: You can connect with: psql 'postgres:///$PGDATABASE?host=$tmpdir/socket' -U ${superuserRole}"
echo "${commandName}: You can tail the logs with: tail -f $tmpdir/db.log"
if test "$_arg_fixtures"; then
log "Loading fixtures under the postgres role..."
psql -U postgres -v PGUSER="$PGUSER" -v ON_ERROR_STOP=1 -f "$_arg_fixtures" >> "$setuplog"
log "Done. Running command..."
fi

("$_arg_command" "''${_arg_leftovers[@]}")
'';
Expand Down
Loading