Remove "bug" & "fixed" parameters, and reimplement functionality with…

… a blacklist (#638)

Sytest has a feature that allows you to mark a test as expected fail, along with a bug ID with the reasoning for it. This is nice, but really only works in the context of a single homeserver, where tests were being disabled for everyone because something didn't work in Synapse.

Since homeserver-specific test white/blacklist was added, we can recreate the functionality of the bug parameter using a blacklist instead.

This PR pairs with matrix-org/synapse#5611

docker/synapse_sytest.sh

@@ -82,7 +82,7 @@ fi
 >&2 echo "+++ Running tests"
 
 RUN_TESTS=(
-    perl -I "$SYTEST_LIB" ./run-tests.pl --python=/venv/bin/python --synapse-directory=/src --coverage -O tap --all
+    perl -I "$SYTEST_LIB" ./run-tests.pl --python=/venv/bin/python --synapse-directory=/src -B /src/sytest-blacklist --coverage -O tap --all
 )
 
 TEST_STATUS=0

run-tests.pl

@@ -55,8 +55,6 @@
 # and the like.
 our $TEST_RUN_ID = strftime( '%Y%m%d_%H%M%S', gmtime() );
 
-my %FIXED_BUGS;
-
 my $STOP_ON_FAIL;
 my $SERVER_IMPL = undef;
 
@@ -107,8 +105,6 @@
 
    'p|port-range=s' => \(my $PORT_RANGE = "8800:8899"),
 
-   'F|fixed=s' => sub { $FIXED_BUGS{$_}++ for split m/,/, $_[1] },
-
    'h|help' => sub { usage(0) },
 ) or usage(1);
 
@@ -175,9 +171,6 @@ sub usage
 
    -p, --port-range START:MAX   - pool of TCP ports to allocate from
 
-   -F, --fixed BUGS             - bug names that are expected to be fixed
-                                  (ignores 'bug' declarations with these names)
-
 .
    write STDERR;
 
@@ -583,10 +576,6 @@ sub _push_test
 {
    my ( $filename, $multi, $name, %params ) = @_;
 
-   # We expect this test to fail if it's declared to be dependent on a bug that
-   # is not yet fixed
-   $params{expect_fail}++ if $params{bug} and not $FIXED_BUGS{ $params{bug} };
-
    if( %only_files and not exists $only_files{$filename} ) {
       $proven{$_} = PRESUMED for @{ $params{proves} // [] };
       return;

tests/30rooms/04messages.pl

@@ -154,9 +154,6 @@
    requires => [ $senduser_fixture, $remote_fixture, $room_fixture,
                 qw( can_receive_room_message_locally )],
 
-   # this test frequently times out for unknown reasons
-   bug => "synapse#1679",
-
    do => sub {
       my ( $senduser, $remote_user, $room_id ) = @_;
 

tests/30rooms/13guestaccess.pl

@@ -489,8 +489,6 @@
 test "Guest users can accept invites to private rooms over federation",
    requires => [ remote_user_fixture(), guest_user_fixture() ],
 
-   bug => "synapse#2065",
-
    do => sub {
       my ( $remote_user, $local_guest ) = @_;
 

tests/31sync/15lazy-members.pl

@@ -202,8 +202,6 @@
    requires => [ local_user_fixtures( 4 ),
                  qw( can_sync ) ],
 
-   bug => "vector-im/riot-web#7211",
-
    check => sub {
       my ( $alice, $bob, $charlie, $dave ) = @_;
 

tests/40presence.pl

@@ -124,9 +124,6 @@
    requires => [ local_user_fixture(),
                  qw( can_initial_sync )],
 
-   # this test fails sometimes. Disable it for now to avoid red-light fatigue.
-   bug => "synapse#1658",
-
    do => sub {
       my ( $user ) = @_;
 

tests/41end-to-end-keys/01-upload-key.pl

@@ -38,8 +38,6 @@
 test "Should reject keys claiming to belong to a different user",
    requires => [ $fixture ],
 
-   bug => "synapse#1396",
-
    do => sub {
       my ( $user ) = @_;
 

tests/52user-directory/01public.pl

@@ -151,9 +151,6 @@
    multi_test "Users appear/disappear from directory when $type are changed",
       requires => [ local_user_fixtures( 2 ) ],
 
-      # this test is currently flaky due to a synapse bug
-      bug => "synapse#2306",
-
       check => sub {
          my ( $creator, $user ) = @_;
 
@@ -249,9 +246,6 @@
 multi_test "Users stay in directory when join_rules are changed but history_visibility is world_readable",
    requires => [ local_user_fixtures( 2 ) ],
 
-   # this test is currently flaky due to a synapse bug
-   bug => "synapse#2306",
-
    check => sub {
       my ( $creator, $user ) = @_;
 

tests/61push/07_set_enabled.pl

@@ -60,8 +60,6 @@ sub check_enable_disable_rule {
 test "Enabling an unknown default rule fails with 404",
    requires => [ local_user_fixture() ],
 
-   bug => "SYN-676",
-
    check => sub {
       my ( $user ) = @_;
       matrix_set_push_rule_enabled(

tests/90jira/SYN-115.pl

@@ -4,10 +4,6 @@
    requires => [ local_user_fixture(), remote_user_fixture( with_events => 1 ),
                  qw( can_create_private_room )],
 
-   # this test fails intermittently on dendron-fronted builds, for unknown
-   # reasons.
-   bug => 'synapse#1663',
-
    do => sub {
       my ( $alice, $bob ) = @_;