Implementation-specific tests (#981)

* Mark admin tests as Synapse-specific

* Match just the first part of the implementation name

* Let's try that again

* Declare tokens

* Scalar vs list etc

* Invert the check

* Fat-fingered

* Less magic

* Define implementation_name for all

* whois is actually in the spec

* Add commented stub, try without implementation_name in ViaHaproxy/Monolith

Author: neilalexander

DEVELOP.rst

@@ -55,6 +55,10 @@ is described in more detail in the following sections.
     deprecated in the spec. These tests will be ignored automatically if
     the ``--exclude-deprecated`` flag is provided to Sytest. 
 
+``implementation_specific``
+    A string to specify that the test should only be run for certain homeserver
+    implementations, e.g. `"Synapse"`, `"Dendrite"` etc. 
+
 A call to ``test`` is a simplified version of ``multi_test`` which produces
 only a single line of test output indicating success or failure automatically.
 A call to ``multi_test`` can make use of additional functions within the body

lib/SyTest/HomeserverFactory.pm

@@ -38,6 +38,14 @@ sub new
 
 sub _init {}
 
+# Returns the generic simple name of the implementation, e.g. "synapse",
+# "dendrite" etc. This is used when determining whether to run any
+# implementation-specific tests.
+sub implementation_name
+{
+    return "";
+}
+
 # returns a list of (name => action) pairs suitable for
 # inclusion in the GetOptions argument list
 sub get_options

lib/SyTest/HomeserverFactory/Dendrite.pm

@@ -31,6 +31,11 @@ sub _init
    $self->SUPER::_init( @_ );
 }
 
+sub implementation_name
+{
+   return "dendrite";
+}
+
 sub get_options
 {
    my $self = shift;

lib/SyTest/HomeserverFactory/Manual.pm

@@ -31,6 +31,11 @@ sub _init
    $self->SUPER::_init( @_ );
 }
 
+sub implementation_name
+{
+   return "manual";
+}
+
 sub get_options
 {
    my $self = shift;

lib/SyTest/HomeserverFactory/Synapse.pm

@@ -38,6 +38,11 @@ sub _init
    $self->SUPER::_init( @_ );
 }
 
+sub implementation_name
+{
+   return "synapse";
+}
+
 sub get_options
 {
    my $self = shift;

run-tests.pl

@@ -622,6 +622,12 @@ sub _push_test
        }
    }
 
+   if( exists $params{implementation_specific} ) {
+       if ( $HS_FACTORY->implementation_name() ne $params{implementation_specific} ) {
+           return;
+       }
+   }
+
    push @TESTS, Test( $filename, $name, $multi,
       @params{qw( expect_fail proves requires check do timeout )} );
 }

tests/10apidoc/01register.pl

@@ -318,6 +318,7 @@ sub shared_secret_tests {
 
    test "POST $ep_name with shared secret",
       requires => [ $main::API_CLIENTS[0], localpart_fixture() ],
+      implementation_specific => "synapse",
 
       proves => [qw( can_register_with_secret )],
 
@@ -334,6 +335,7 @@ sub shared_secret_tests {
 
    test "POST $ep_name admin with shared secret",
       requires => [ $main::API_CLIENTS[0], localpart_fixture() ],
+      implementation_specific => "synapse",
 
       do => sub {
          my ( $http, $uid ) = @_;
@@ -349,6 +351,7 @@ sub shared_secret_tests {
 
    test "POST $ep_name with shared secret downcases capitals",
       requires => [ $main::API_CLIENTS[0], localpart_fixture() ],
+      implementation_specific => "synapse",
 
       proves => [qw( can_register_with_secret )],
 
@@ -365,6 +368,7 @@ sub shared_secret_tests {
 
    test "POST $ep_name with shared secret disallows symbols",
       requires => [ $main::API_CLIENTS[0] ],
+      implementation_specific => "synapse",
 
       proves => [qw( can_register_with_secret )],
 

tests/48admin.pl

@@ -69,6 +69,7 @@ sub await_purge_complete {
 
 test "/purge_history",
    requires => [ local_admin_fixture(), local_user_and_room_fixtures() ],
+   implementation_specific => "synapse",
 
    do => sub {
       my ( $admin, $user, $room_id ) = @_;
@@ -149,6 +150,7 @@ sub await_purge_complete {
 
 test "/purge_history by ts",
    requires => [ local_admin_fixture(), local_user_and_room_fixtures() ],
+   implementation_specific => "synapse",
 
    do => sub {
       my ( $admin, $user, $room_id ) = @_;
@@ -222,6 +224,7 @@ sub await_purge_complete {
 test "Can backfill purged history",
    requires => [ local_admin_fixture(), local_user_and_room_fixtures(),
                  remote_user_fixture(), qw( can_paginate_room_remotely ) ],
+   implementation_specific => "synapse",
 
    # this test is a bit slow.
    timeout => 50,
@@ -358,6 +361,7 @@ sub await_purge_complete {
 multi_test "Shutdown room",
    requires => [ local_admin_fixture(), local_user_fixtures( 2 ), remote_user_fixture(),
       room_alias_name_fixture() ],
+   implementation_specific => "synapse",
 
    do => sub {
       my ( $admin, $user, $dummy_user, $remote_user, $room_alias_name ) = @_;