Add Complement dockerfile and move sytest dir

6 files changed, 1344 insertions, 0 deletions

diff --git a/tests/Complement.Dockerfile b/tests/Complement.Dockerfile
new file mode 100644
index 0000000..56e4bf7
--- /dev/null
+++ b/tests/Complement.Dockerfile
@@ -0,0 +1,10 @@
+FROM valkum/docker-rust-ci:latest
+WORKDIR /build
+
+COPY . .
+RUN cargo build
+
+ENV SERVER_NAME=localhost
+EXPOSE 14004 8448
+
+CMD sed "s/server_name: your.server.name/server_name: ${SERVER_NAME}/g" Rocket-example.toml Rocket.toml && ./target/debug/conduit
\ No newline at end of file
diff --git a/tests/sytest/are-we-synapse-yet.list b/tests/sytest/are-we-synapse-yet.list
new file mode 100644
index 0000000..cdc280a
--- /dev/null
+++ b/tests/sytest/are-we-synapse-yet.list
@@ -0,0 +1,836 @@
+reg GET /register yields a set of flows
+reg POST /register can create a user
+reg POST /register downcases capitals in usernames
+reg POST /register returns the same device_id as that in the request
+reg POST /register rejects registration of usernames with '!'
+reg POST /register rejects registration of usernames with '"'
+reg POST /register rejects registration of usernames with ':'
+reg POST /register rejects registration of usernames with '?'
+reg POST /register rejects registration of usernames with '\'
+reg POST /register rejects registration of usernames with '@'
+reg POST /register rejects registration of usernames with '['
+reg POST /register rejects registration of usernames with ']'
+reg POST /register rejects registration of usernames with '{'
+reg POST /register rejects registration of usernames with '|'
+reg POST /register rejects registration of usernames with '}'
+reg POST /register rejects registration of usernames with '£'
+reg POST /register rejects registration of usernames with 'é'
+reg POST /register rejects registration of usernames with '\n'
+reg POST /register rejects registration of usernames with '''
+reg POST /r0/admin/register with shared secret
+reg POST /r0/admin/register admin with shared secret
+reg POST /r0/admin/register with shared secret downcases capitals
+reg POST /r0/admin/register with shared secret disallows symbols
+reg POST rejects invalid utf-8 in JSON
+log GET /login yields a set of flows
+log POST /login can log in as a user
+log POST /login returns the same device_id as that in the request
+log POST /login can log in as a user with just the local part of the id
+log POST /login as non-existing user is rejected
+log POST /login wrong password is rejected
+log Interactive authentication types include SSO
+log Can perform interactive authentication with SSO
+log The user must be consistent through an interactive authentication session with SSO
+log The operation must be consistent through an interactive authentication session
+v1s GET /events initially
+v1s GET /initialSync initially
+csa Version responds 200 OK with valid structure
+pro PUT /profile/:user_id/displayname sets my name
+pro GET /profile/:user_id/displayname publicly accessible
+pro PUT /profile/:user_id/avatar_url sets my avatar
+pro GET /profile/:user_id/avatar_url publicly accessible
+dev GET /device/{deviceId}
+dev GET /device/{deviceId} gives a 404 for unknown devices
+dev GET /devices
+dev PUT /device/{deviceId} updates device fields
+dev PUT /device/{deviceId} gives a 404 for unknown devices
+dev DELETE /device/{deviceId}
+dev DELETE /device/{deviceId} requires UI auth user to match device owner
+dev DELETE /device/{deviceId} with no body gives a 401
+dev The deleted device must be consistent through an interactive auth session
+dev Users receive device_list updates for their own devices
+pre GET /presence/:user_id/status fetches initial status
+pre PUT /presence/:user_id/status updates my presence
+crm POST /createRoom makes a public room
+crm POST /createRoom makes a private room
+crm POST /createRoom makes a private room with invites
+crm POST /createRoom makes a room with a name
+crm POST /createRoom makes a room with a topic
+syn Can /sync newly created room
+crm POST /createRoom creates a room with the given version
+crm POST /createRoom rejects attempts to create rooms with numeric versions
+crm POST /createRoom rejects attempts to create rooms with unknown versions
+crm POST /createRoom ignores attempts to set the room version via creation_content
+mem GET /rooms/:room_id/state/m.room.member/:user_id fetches my membership
+mem GET /rooms/:room_id/state/m.room.member/:user_id?format=event fetches my membership event
+rst GET /rooms/:room_id/state/m.room.power_levels fetches powerlevels
+mem GET /rooms/:room_id/joined_members fetches my membership
+v1s GET /rooms/:room_id/initialSync fetches initial sync state
+pub GET /publicRooms lists newly-created room
+ali GET /directory/room/:room_alias yields room ID
+mem GET /joined_rooms lists newly-created room
+rst POST /rooms/:room_id/state/m.room.name sets name
+rst GET /rooms/:room_id/state/m.room.name gets name
+rst POST /rooms/:room_id/state/m.room.topic sets topic
+rst GET /rooms/:room_id/state/m.room.topic gets topic
+rst GET /rooms/:room_id/state fetches entire room state
+crm POST /createRoom with creation content
+ali PUT /directory/room/:room_alias creates alias
+nsp GET /rooms/:room_id/aliases lists aliases
+jon POST /rooms/:room_id/join can join a room
+jon POST /join/:room_alias can join a room
+jon POST /join/:room_id can join a room
+jon POST /join/:room_id can join a room with custom content
+jon POST /join/:room_alias can join a room with custom content
+lev POST /rooms/:room_id/leave can leave a room
+inv POST /rooms/:room_id/invite can send an invite
+ban POST /rooms/:room_id/ban can ban a user
+snd POST /rooms/:room_id/send/:event_type sends a message
+snd PUT /rooms/:room_id/send/:event_type/:txn_id sends a message
+snd PUT /rooms/:room_id/send/:event_type/:txn_id deduplicates the same txn id
+get GET /rooms/:room_id/messages returns a message
+get GET /rooms/:room_id/messages lazy loads members correctly
+typ PUT /rooms/:room_id/typing/:user_id sets typing notification
+rst GET /rooms/:room_id/state/m.room.power_levels can fetch levels
+rst PUT /rooms/:room_id/state/m.room.power_levels can set levels
+rst PUT power_levels should not explode if the old power levels were empty
+rst Both GET and PUT work
+rct POST /rooms/:room_id/receipt can create receipts
+red POST /rooms/:room_id/read_markers can create read marker
+med POST /media/v1/upload can create an upload
+med GET /media/v1/download can fetch the value again
+cap GET /capabilities is present and well formed for registered user
+cap GET /r0/capabilities is not public
+reg Register with a recaptcha
+reg registration is idempotent, without username specified
+reg registration is idempotent, with username specified
+reg registration remembers parameters
+reg registration accepts non-ascii passwords
+reg registration with inhibit_login inhibits login
+reg User signups are forbidden from starting with '_'
+reg Can register using an email address
+log Can login with 3pid and password using m.login.password
+log login types include SSO
+log /login/cas/redirect redirects if the old m.login.cas login type is listed
+log Can login with new user via CAS
+lox Can logout current device
+lox Can logout all devices
+lox Request to logout with invalid an access token is rejected
+lox Request to logout without an access token is rejected
+log After changing password, can't log in with old password
+log After changing password, can log in with new password
+log After changing password, existing session still works
+log After changing password, a different session no longer works by default
+log After changing password, different sessions can optionally be kept
+psh Pushers created with a different access token are deleted on password change
+psh Pushers created with a the same access token are not deleted on password change
+acc Can deactivate account
+acc Can't deactivate account with wrong password
+acc After deactivating account, can't log in with password
+acc After deactivating account, can't log in with an email
+v1s initialSync sees my presence status
+pre Presence change reports an event to myself
+pre Friends presence changes reports events
+crm Room creation reports m.room.create to myself
+crm Room creation reports m.room.member to myself
+rst Setting room topic reports m.room.topic to myself
+v1s Global initialSync
+v1s Global initialSync with limit=0 gives no messages
+v1s Room initialSync
+v1s Room initialSync with limit=0 gives no messages
+rst Setting state twice is idempotent
+jon Joining room twice is idempotent
+syn New room members see their own join event
+v1s New room members see existing users' presence in room initialSync
+syn Existing members see new members' join events
+syn Existing members see new members' presence
+v1s All room members see all room members' presence in global initialSync
+f,jon Remote users can join room by alias
+syn New room members see their own join event
+v1s New room members see existing members' presence in room initialSync
+syn Existing members see new members' join events
+syn Existing members see new member's presence
+v1s New room members see first user's profile information in global initialSync
+v1s New room members see first user's profile information in per-room initialSync
+f,jon Remote users may not join unfederated rooms
+syn Local room members see posted message events
+v1s Fetching eventstream a second time doesn't yield the message again
+syn Local non-members don't see posted message events
+get Local room members can get room messages
+f,syn Remote room members also see posted message events
+f,get Remote room members can get room messages
+get Message history can be paginated
+f,get Message history can be paginated over federation
+eph Ephemeral messages received from clients are correctly expired
+ali Room aliases can contain Unicode
+f,ali Remote room alias queries can handle Unicode
+ali Canonical alias can be set
+ali Canonical alias can include alt_aliases
+ali Regular users can add and delete aliases in the default room configuration
+ali Regular users can add and delete aliases when m.room.aliases is restricted
+ali Deleting a non-existent alias should return a 404
+ali Users can't delete other's aliases
+ali Users with sufficient power-level can delete other's aliases
+ali Can delete canonical alias
+ali Alias creators can delete alias with no ops
+ali Alias creators can delete canonical alias with no ops
+ali Only room members can list aliases of a room
+inv Can invite users to invite-only rooms
+inv Uninvited users cannot join the room
+inv Invited user can reject invite
+f,inv Invited user can reject invite over federation
+f,inv Invited user can reject invite over federation several times
+inv Invited user can reject invite for empty room
+f,inv Invited user can reject invite over federation for empty room
+inv Invited user can reject local invite after originator leaves
+inv Invited user can see room metadata
+f,inv Remote invited user can see room metadata
+inv Users cannot invite themselves to a room
+inv Users cannot invite a user that is already in the room
+ban Banned user is kicked and may not rejoin until unbanned
+f,ban Remote banned user is kicked and may not rejoin until unbanned
+ban 'ban' event respects room powerlevel
+plv setting 'm.room.name' respects room powerlevel
+plv setting 'm.room.power_levels' respects room powerlevel (2 subtests)
+plv Unprivileged users can set m.room.topic if it only needs level 0
+plv Users cannot set ban powerlevel higher than their own (2 subtests)
+plv Users cannot set kick powerlevel higher than their own (2 subtests)
+plv Users cannot set redact powerlevel higher than their own (2 subtests)
+v1s Check that event streams started after a client joined a room work (SYT-1)
+v1s Event stream catches up fully after many messages
+xxx POST /rooms/:room_id/redact/:event_id as power user redacts message
+xxx POST /rooms/:room_id/redact/:event_id as original message sender redacts message
+xxx POST /rooms/:room_id/redact/:event_id as random user does not redact message
+xxx POST /redact disallows redaction of event in different room
+xxx Redaction of a redaction redacts the redaction reason
+v1s A departed room is still included in /initialSync (SPEC-216)
+v1s Can get rooms/{roomId}/initialSync for a departed room (SPEC-216)
+rst Can get rooms/{roomId}/state for a departed room (SPEC-216)
+mem Can get rooms/{roomId}/members for a departed room (SPEC-216)
+get Can get rooms/{roomId}/messages for a departed room (SPEC-216)
+rst Can get 'm.room.name' state for a departed room (SPEC-216)
+syn Getting messages going forward is limited for a departed room (SPEC-216)
+3pd Can invite existing 3pid
+3pd Can invite existing 3pid with no ops into a private room
+3pd Can invite existing 3pid in createRoom
+3pd Can invite unbound 3pid
+f,3pd Can invite unbound 3pid over federation
+3pd Can invite unbound 3pid with no ops into a private room
+f,3pd Can invite unbound 3pid over federation with no ops into a private room
+f,3pd Can invite unbound 3pid over federation with users from both servers
+3pd Can accept unbound 3pid invite after inviter leaves
+3pd Can accept third party invite with /join
+3pd 3pid invite join with wrong but valid signature are rejected
+3pd 3pid invite join valid signature but revoked keys are rejected
+3pd 3pid invite join valid signature but unreachable ID server are rejected
+gst Guest user cannot call /events globally
+gst Guest users can join guest_access rooms
+gst Guest users can send messages to guest_access rooms if joined
+gst Guest user calling /events doesn't tightloop
+gst Guest users are kicked from guest_access rooms on revocation of guest_access
+gst Guest user can set display names
+gst Guest users are kicked from guest_access rooms on revocation of guest_access over federation
+gst Guest user can upgrade to fully featured user
+gst Guest user cannot upgrade other users
+pub GET /publicRooms lists rooms
+pub GET /publicRooms includes avatar URLs
+gst Guest users can accept invites to private rooms over federation
+gst Guest users denied access over federation if guest access prohibited
+mem Room members can override their displayname on a room-specific basis
+mem Room members can join a room with an overridden displayname
+mem Users cannot kick users from a room they are not in
+mem Users cannot kick users who have already left a room
+typ Typing notification sent to local room members
+f,typ Typing notifications also sent to remote room members
+typ Typing can be explicitly stopped
+rct Read receipts are visible to /initialSync
+rct Read receipts are sent as events
+rct Receipts must be m.read
+pro displayname updates affect room member events
+pro avatar_url updates affect room member events
+gst m.room.history_visibility == "world_readable" allows/forbids appropriately for Guest users
+gst m.room.history_visibility == "shared" allows/forbids appropriately for Guest users
+gst m.room.history_visibility == "invited" allows/forbids appropriately for Guest users
+gst m.room.history_visibility == "joined" allows/forbids appropriately for Guest users
+gst m.room.history_visibility == "default" allows/forbids appropriately for Guest users
+gst Guest non-joined user cannot call /events on shared room
+gst Guest non-joined user cannot call /events on invited room
+gst Guest non-joined user cannot call /events on joined room
+gst Guest non-joined user cannot call /events on default room
+gst Guest non-joined user can call /events on world_readable room
+gst Guest non-joined users can get state for world_readable rooms
+gst Guest non-joined users can get individual state for world_readable rooms
+gst Guest non-joined users cannot room initalSync for non-world_readable rooms
+gst Guest non-joined users can room initialSync for world_readable rooms
+gst Guest non-joined users can get individual state for world_readable rooms after leaving
+gst Guest non-joined users cannot send messages to guest_access rooms if not joined
+gst Guest users can sync from world_readable guest_access rooms if joined
+gst Guest users can sync from shared guest_access rooms if joined
+gst Guest users can sync from invited guest_access rooms if joined
+gst Guest users can sync from joined guest_access rooms if joined
+gst Guest users can sync from default guest_access rooms if joined
+ath m.room.history_visibility == "world_readable" allows/forbids appropriately for Real users
+ath m.room.history_visibility == "shared" allows/forbids appropriately for Real users
+ath m.room.history_visibility == "invited" allows/forbids appropriately for Real users
+ath m.room.history_visibility == "joined" allows/forbids appropriately for Real users
+ath m.room.history_visibility == "default" allows/forbids appropriately for Real users
+ath Real non-joined user cannot call /events on shared room
+ath Real non-joined user cannot call /events on invited room
+ath Real non-joined user cannot call /events on joined room
+ath Real non-joined user cannot call /events on default room
+ath Real non-joined user can call /events on world_readable room
+ath Real non-joined users can get state for world_readable rooms
+ath Real non-joined users can get individual state for world_readable rooms
+ath Real non-joined users cannot room initalSync for non-world_readable rooms
+ath Real non-joined users can room initialSync for world_readable rooms
+ath Real non-joined users can get individual state for world_readable rooms after leaving
+ath Real non-joined users cannot send messages to guest_access rooms if not joined
+ath Real users can sync from world_readable guest_access rooms if joined
+ath Real users can sync from shared guest_access rooms if joined
+ath Real users can sync from invited guest_access rooms if joined
+ath Real users can sync from joined guest_access rooms if joined
+ath Real users can sync from default guest_access rooms if joined
+ath Only see history_visibility changes on boundaries
+f,ath Backfill works correctly with history visibility set to joined
+fgt Forgotten room messages cannot be paginated
+fgt Forgetting room does not show up in v2 /sync
+fgt Can forget room you've been kicked from
+fgt Can't forget room you're still in
+mem Can re-join room if re-invited
+ath Only original members of the room can see messages from erased users
+mem /joined_rooms returns only joined rooms
+mem /joined_members return joined members
+ctx /context/ on joined room works
+ctx /context/ on non world readable room does not work
+ctx /context/ returns correct number of events
+ctx /context/ with lazy_load_members filter works
+get /event/ on joined room works
+get /event/ on non world readable room does not work
+get /event/ does not allow access to events before the user joined
+mem Can get rooms/{roomId}/members
+mem Can get rooms/{roomId}/members at a given point
+mem Can filter rooms/{roomId}/members
+upg /upgrade creates a new room
+upg /upgrade should preserve room visibility for public rooms
+upg /upgrade should preserve room visibility for private rooms
+upg /upgrade copies >100 power levels to the new room
+upg /upgrade copies the power levels to the new room
+upg /upgrade preserves the power level of the upgrading user in old and new rooms
+upg /upgrade copies important state to the new room
+upg /upgrade copies ban events to the new room
+upg local user has push rules copied to upgraded room
+f,upg remote user has push rules copied to upgraded room
+upg /upgrade moves aliases to the new room
+upg /upgrade moves remote aliases to the new room
+upg /upgrade preserves direct room state
+upg /upgrade preserves room federation ability
+upg /upgrade restricts power levels in the old room
+upg /upgrade restricts power levels in the old room when the old PLs are unusual
+upg /upgrade to an unknown version is rejected
+upg /upgrade is rejected if the user can't send state events
+upg /upgrade of a bogus room fails gracefully
+upg Cannot send tombstone event that points to the same room
+f,upg Local and remote users' homeservers remove a room from their public directory on upgrade
+rst Name/topic keys are correct
+f,pub Can get remote public room list
+pub Can paginate public room list
+pub Can search public room list
+syn Can create filter
+syn Can download filter
+syn Can sync
+syn Can sync a joined room
+syn Full state sync includes joined rooms
+syn Newly joined room is included in an incremental sync
+syn Newly joined room has correct timeline in incremental sync
+syn Newly joined room includes presence in incremental sync
+syn Get presence for newly joined members in incremental sync
+syn Can sync a room with a single message
+syn Can sync a room with a message with a transaction id
+syn A message sent after an initial sync appears in the timeline of an incremental sync.
+syn A filtered timeline reaches its limit
+syn Syncing a new room with a large timeline limit isn't limited
+syn A full_state incremental update returns only recent timeline
+syn A prev_batch token can be used in the v1 messages API
+syn A next_batch token can be used in the v1 messages API
+syn User sees their own presence in a sync
+syn User is offline if they set_presence=offline in their sync
+syn User sees updates to presence from other users in the incremental sync.
+syn State is included in the timeline in the initial sync
+f,syn State from remote users is included in the state in the initial sync
+syn Changes to state are included in an incremental sync
+syn Changes to state are included in an gapped incremental sync
+f,syn State from remote users is included in the timeline in an incremental sync
+syn A full_state incremental update returns all state
+syn When user joins a room the state is included in the next sync
+syn A change to displayname should not result in a full state sync
+syn A change to displayname should appear in incremental /sync
+syn When user joins a room the state is included in a gapped sync
+syn When user joins and leaves a room in the same batch, the full state is still included in the next sync
+syn Current state appears in timeline in private history
+syn Current state appears in timeline in private history with many messages before
+syn Current state appears in timeline in private history with many messages after
+syn Rooms a user is invited to appear in an initial sync
+syn Rooms a user is invited to appear in an incremental sync
+syn Newly joined room is included in an incremental sync after invite
+syn Sync can be polled for updates
+syn Sync is woken up for leaves
+syn Left rooms appear in the leave section of sync
+syn Newly left rooms appear in the leave section of incremental sync
+syn We should see our own leave event, even if history_visibility is restricted (SYN-662)
+syn We should see our own leave event when rejecting an invite, even if history_visibility is restricted (riot-web/3462)
+syn Newly left rooms appear in the leave section of gapped sync
+syn Previously left rooms don't appear in the leave section of sync
+syn Left rooms appear in the leave section of full state sync
+syn Archived rooms only contain history from before the user left
+syn Banned rooms appear in the leave section of sync
+syn Newly banned rooms appear in the leave section of incremental sync
+syn Newly banned rooms appear in the leave section of incremental sync
+syn Typing events appear in initial sync
+syn Typing events appear in incremental sync
+syn Typing events appear in gapped sync
+syn Read receipts appear in initial v2 /sync
+syn New read receipts appear in incremental v2 /sync
+syn Can pass a JSON filter as a query parameter
+syn Can request federation format via the filter
+syn Read markers appear in incremental v2 /sync
+syn Read markers appear in initial v2 /sync
+syn Read markers can be updated
+syn Lazy loading parameters in the filter are strictly boolean
+syn The only membership state included in an initial sync is for all the senders in the timeline
+syn The only membership state included in an incremental sync is for senders in the timeline
+syn The only membership state included in a gapped incremental sync is for senders in the timeline
+syn Gapped incremental syncs include all state changes
+syn Old leaves are present in gapped incremental syncs
+syn Leaves are present in non-gapped incremental syncs
+syn Old members are included in gappy incr LL sync if they start speaking
+syn Members from the gap are included in gappy incr LL sync
+syn We don't send redundant membership state across incremental syncs by default
+syn We do send redundant membership state across incremental syncs if asked
+syn Unnamed room comes with a name summary
+syn Named room comes with just joined member count summary
+syn Room summary only has 5 heroes
+syn Room summary counts change when membership changes
+rmv User can create and send/receive messages in a room with version 1
+rmv User can create and send/receive messages in a room with version 1 (2 subtests)
+rmv local user can join room with version 1
+rmv User can invite local user to room with version 1
+rmv remote user can join room with version 1
+rmv User can invite remote user to room with version 1
+rmv Remote user can backfill in a room with version 1
+rmv Can reject invites over federation for rooms with version 1
+rmv Can receive redactions from regular users over federation in room version 1
+rmv User can create and send/receive messages in a room with version 2
+rmv User can create and send/receive messages in a room with version 2 (2 subtests)
+rmv local user can join room with version 2
+rmv User can invite local user to room with version 2
+rmv remote user can join room with version 2
+rmv User can invite remote user to room with version 2
+rmv Remote user can backfill in a room with version 2
+rmv Can reject invites over federation for rooms with version 2
+rmv Can receive redactions from regular users over federation in room version 2
+rmv User can create and send/receive messages in a room with version 3
+rmv User can create and send/receive messages in a room with version 3 (2 subtests)
+rmv local user can join room with version 3
+rmv User can invite local user to room with version 3
+rmv remote user can join room with version 3
+rmv User can invite remote user to room with version 3
+rmv Remote user can backfill in a room with version 3
+rmv Can reject invites over federation for rooms with version 3
+rmv Can receive redactions from regular users over federation in room version 3
+rmv User can create and send/receive messages in a room with version 4
+rmv User can create and send/receive messages in a room with version 4 (2 subtests)
+rmv local user can join room with version 4
+rmv User can invite local user to room with version 4
+rmv remote user can join room with version 4
+rmv User can invite remote user to room with version 4
+rmv Remote user can backfill in a room with version 4
+rmv Can reject invites over federation for rooms with version 4
+rmv Can receive redactions from regular users over federation in room version 4
+rmv User can create and send/receive messages in a room with version 5
+rmv User can create and send/receive messages in a room with version 5 (2 subtests)
+rmv local user can join room with version 5
+rmv User can invite local user to room with version 5
+rmv remote user can join room with version 5
+rmv User can invite remote user to room with version 5
+rmv Remote user can backfill in a room with version 5
+rmv Can reject invites over federation for rooms with version 5
+rmv Can receive redactions from regular users over federation in room version 5
+pre Presence changes are reported to local room members
+f,pre Presence changes are also reported to remote room members
+pre Presence changes to UNAVAILABLE are reported to local room members
+f,pre Presence changes to UNAVAILABLE are reported to remote room members
+v1s Newly created users see their own presence in /initialSync (SYT-34)
+dvk Can upload device keys
+dvk Should reject keys claiming to belong to a different user
+dvk Can query device keys using POST
+dvk Can query specific device keys using POST
+dvk query for user with no keys returns empty key dict
+dvk Can claim one time key using POST
+f,dvk Can query remote device keys using POST
+f,dvk Can claim remote one time key using POST
+dvk Local device key changes appear in v2 /sync
+dvk Local new device changes appear in v2 /sync
+dvk Local delete device changes appear in v2 /sync
+dvk Local update device changes appear in v2 /sync
+dvk Can query remote device keys using POST after notification
+f,dev Device deletion propagates over federation
+f,dev If remote user leaves room, changes device and rejoins we see update in sync
+f,dev If remote user leaves room we no longer receive device updates
+dvk Local device key changes appear in /keys/changes
+dvk New users appear in /keys/changes
+f,dvk If remote user leaves room, changes device and rejoins we see update in /keys/changes
+dvk Get left notifs in sync and /keys/changes when other user leaves
+dvk Get left notifs for other users in sync and /keys/changes when user leaves
+f,dvk If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes
+dvk Can create backup version
+dvk Can update backup version
+dvk Responds correctly when backup is empty
+dvk Can backup keys
+dvk Can update keys with better versions
+dvk Will not update keys with worse versions
+dvk Will not back up to an old backup version
+dvk Can delete backup
+dvk Deleted & recreated backups are empty
+dvk Can create more than 10 backup versions
+dvk Can upload self-signing keys
+dvk Fails to upload self-signing keys with no auth
+dvk Fails to upload self-signing key without master key
+dvk Changing master key notifies local users
+dvk Changing user-signing key notifies local users
+f,dvk can fetch self-signing keys over federation
+f,dvk uploading self-signing key notifies over federation
+f,dvk uploading signed devices gets propagated over federation
+tag Can add tag
+tag Can remove tag
+tag Can list tags for a room
+v1s Tags appear in the v1 /events stream
+v1s Tags appear in the v1 /initalSync
+v1s Tags appear in the v1 room initial sync
+tag Tags appear in an initial v2 /sync
+tag Newly updated tags appear in an incremental v2 /sync
+tag Deleted tags appear in an incremental v2 /sync
+tag local user has tags copied to the new room
+f,tag remote user has tags copied to the new room
+sch Can search for an event by body
+sch Can get context around search results
+sch Can back-paginate search results
+sch Search works across an upgraded room and its predecessor
+sch Search results with rank ordering do not include redacted events
+sch Search results with recent ordering do not include redacted events
+acc Can add account data
+acc Can add account data to room
+acc Can get account data without syncing
+acc Can get room account data without syncing
+v1s Latest account data comes down in /initialSync
+v1s Latest account data comes down in room initialSync
+v1s Account data appears in v1 /events stream
+v1s Room account data appears in v1 /events stream
+acc Latest account data appears in v2 /sync
+acc New account data appears in incremental v2 /sync
+oid Can generate a openid access_token that can be exchanged for information about a user
+oid Invalid openid access tokens are rejected
+oid Requests to userinfo without access tokens are rejected
+std Can send a message directly to a device using PUT /sendToDevice
+std Can recv a device message using /sync
+std Can recv device messages until they are acknowledged
+std Device messages with the same txn_id are deduplicated
+std Device messages wake up /sync
+std Can recv device messages over federation
+std Device messages over federation wake up /sync
+std Can send messages with a wildcard device id
+std Can send messages with a wildcard device id to two devices
+std Wildcard device messages wake up /sync
+std Wildcard device messages over federation wake up /sync
+adm /whois
+nsp /purge_history
+nsp /purge_history by ts
+nsp Can backfill purged history
+nsp Shutdown room
+ign Ignore user in existing room
+ign Ignore invite in full sync
+ign Ignore invite in incremental sync
+fky Checking local federation server
+fky Federation key API allows unsigned requests for keys
+fky Federation key API can act as a notary server via a GET request
+fky Federation key API can act as a notary server via a POST request
+fky Key notary server should return an expired key if it can't find any others
+fky Key notary server must not overwrite a valid key with a spurious result from the origin server
+fqu Non-numeric ports in server names are rejected
+fqu Outbound federation can query profile data
+fqu Inbound federation can query profile data
+fqu Outbound federation can query room alias directory
+fqu Inbound federation can query room alias directory
+fsj Outbound federation can query v1 /send_join
+fsj Outbound federation can query v2 /send_join
+fmj Outbound federation passes make_join failures through to the client
+fsj Inbound federation can receive v1 /send_join
+fsj Inbound federation can receive v2 /send_join
+fmj Inbound /v1/make_join rejects remote attempts to join local users to rooms
+fsj Inbound /v1/send_join rejects incorrectly-signed joins
+fsj Inbound /v1/send_join rejects joins from other servers
+fau Inbound federation rejects remote attempts to kick local users to rooms
+frv Inbound federation rejects attempts to join v1 rooms from servers without v1 support
+frv Inbound federation rejects attempts to join v2 rooms from servers lacking version support
+frv Inbound federation rejects attempts to join v2 rooms from servers only supporting v1
+frv Inbound federation accepts attempts to join v2 rooms from servers with support
+frv Outbound federation correctly handles unsupported room versions
+frv A pair of servers can establish a join in a v2 room
+fsj Outbound federation rejects send_join responses with no m.room.create event
+frv Outbound federation rejects m.room.create events with an unknown room version
+fsj Event with an invalid signature in the send_join response should not cause room join to fail
+fed Outbound federation can send events
+fed Inbound federation can receive events
+fed Inbound federation can receive redacted events
+fed Ephemeral messages received from servers are correctly expired
+fed Events whose auth_events are in the wrong room do not mess up the room state
+fed Inbound federation can return events
+fed Inbound federation redacts events from erased users
+fme Outbound federation can request missing events
+fme Inbound federation can return missing events for world_readable visibility
+fme Inbound federation can return missing events for shared visibility
+fme Inbound federation can return missing events for invite visibility
+fme Inbound federation can return missing events for joined visibility
+fme outliers whose auth_events are in a different room are correctly rejected
+fbk Outbound federation can backfill events
+fbk Inbound federation can backfill events
+fbk Backfill checks the events requested belong to the room
+fbk Backfilled events whose prev_events are in a different room do not allow cross-room back-pagination
+fiv Outbound federation can send invites via v1 API
+fiv Outbound federation can send invites via v2 API
+fiv Inbound federation can receive invites via v1 API
+fiv Inbound federation can receive invites via v2 API
+fiv Inbound federation can receive invite and reject when remote replies with a 403
+fiv Inbound federation can receive invite and reject when remote replies with a 500
+fiv Inbound federation can receive invite and reject when remote is unreachable
+fiv Inbound federation rejects invites which are not signed by the sender
+fiv Inbound federation can receive invite rejections
+fiv Inbound federation rejects incorrectly-signed invite rejections
+fsl Inbound /v1/send_leave rejects leaves from other servers
+fst Inbound federation can get state for a room
+fst Inbound federation of state requires event_id as a mandatory paramater
+fst Inbound federation can get state_ids for a room
+fst Inbound federation of state_ids requires event_id as a mandatory paramater
+fst  Federation rejects inbound events where the prev_events cannot be found
+fst Room state at a rejected message event is the same as its predecessor
+fst Room state at a rejected state event is the same as its predecessor
+fst Outbound federation requests missing prev_events and then asks for /state_ids and resolves the state
+fst Federation handles empty auth_events in state_ids sanely
+fst Getting state checks the events requested belong to the room
+fst Getting state IDs checks the events requested belong to the room
+fst Should not be able to take over the room by pretending there is no PL event
+fpb Inbound federation can get public room list
+fed Outbound federation sends receipts
+fed Inbound federation rejects receipts from wrong remote
+fed Inbound federation ignores redactions from invalid servers room > v3
+fed An event which redacts an event in a different room should be ignored
+fed An event which redacts itself should be ignored
+fed A pair of events which redact each other should be ignored
+fdk Local device key changes get to remote servers
+fdk Server correctly handles incoming m.device_list_update
+fdk Server correctly resyncs when client query keys and there is no remote cache
+fdk Server correctly resyncs when server leaves and rejoins a room
+fdk Local device key changes get to remote servers with correct prev_id
+fdk Device list doesn't change if remote server is down
+fdk If a device list update goes missing, the server resyncs on the next one
+fst Name/topic keys are correct
+fau Remote servers cannot set power levels in rooms without existing powerlevels
+fau Remote servers should reject attempts by non-creators to set the power levels
+fau Inbound federation rejects typing notifications from wrong remote
+fed Forward extremities remain so even after the next events are populated as outliers
+fau Banned servers cannot send events
+fau Banned servers cannot /make_join
+fau Banned servers cannot /send_join
+fau Banned servers cannot /make_leave
+fau Banned servers cannot /send_leave
+fau Banned servers cannot /invite
+fau Banned servers cannot get room state
+fau Banned servers cannot get room state ids
+fau Banned servers cannot backfill
+fau Banned servers cannot /event_auth
+fau Banned servers cannot get missing events
+fau Server correctly handles transactions that break edu limits
+fau Inbound federation correctly soft fails events
+fau Inbound federation accepts a second soft-failed event
+fau Inbound federation correctly handles soft failed events as extremities
+med Can upload with Unicode file name
+med Can download with Unicode file name locally
+f,med Can download with Unicode file name over federation
+med Alternative server names do not cause a routing loop
+med Can download specifying a different Unicode file name
+med Can upload without a file name
+med Can download without a file name locally
+f,med Can download without a file name over federation
+med Can upload with ASCII file name
+med Can download file 'ascii'
+med Can download file 'name with spaces'
+med Can download file 'name;with;semicolons'
+med Can download specifying a different ASCII file name
+med Can send image in room message
+med Can fetch images in room
+med POSTed media can be thumbnailed
+f,med Remote media can be thumbnailed
+med Test URL preview
+med Can read configuration endpoint
+nsp Can quarantine media in rooms
+udr User appears in user directory
+udr User in private room doesn't appear in user directory
+udr User joining then leaving public room appears and dissappears from directory
+udr Users appear/disappear from directory when join_rules are changed
+udr Users appear/disappear from directory when history_visibility are changed
+udr Users stay in directory when join_rules are changed but history_visibility is world_readable
+f,udr User in remote room doesn't appear in user directory after server left room
+udr User directory correctly update on display name change
+udr User in shared private room does appear in user directory
+udr User in shared private room does appear in user directory until leave
+udr User in dir while user still shares private rooms
+nsp Create group
+nsp Add group rooms
+nsp Remove group rooms
+nsp Get local group profile
+nsp Get local group users
+nsp Add/remove local group rooms
+nsp Get local group summary
+nsp Get remote group profile
+nsp Get remote group users
+nsp Add/remove remote group rooms
+nsp Get remote group summary
+nsp Add local group users
+nsp Remove self from local group
+nsp Remove other from local group
+nsp Add remote group users
+nsp Remove self from remote group
+nsp Listing invited users of a remote group when not a member returns a 403
+nsp Add group category
+nsp Remove group category
+nsp Get group categories
+nsp Add group role
+nsp Remove group role
+nsp Get group roles
+nsp Add room to group summary
+nsp Adding room to group summary keeps room_id when fetching rooms in group
+nsp Adding multiple rooms to group summary have correct order
+nsp Remove room from group summary
+nsp Add room to group summary with category
+nsp Remove room from group summary with category
+nsp Add user to group summary
+nsp Adding multiple users to group summary have correct order
+nsp Remove user from group summary
+nsp Add user to group summary with role
+nsp Remove user from group summary with role
+nsp Local group invites come down sync
+nsp Group creator sees group in sync
+nsp Group creator sees group in initial sync
+nsp Get/set local group publicity
+nsp Bulk get group publicity
+nsp Joinability comes down summary
+nsp Set group joinable and join it
+nsp Group is not joinable by default
+nsp Group is joinable over federation
+nsp Room is transitioned on local and remote groups upon room upgrade
+3pd Can bind 3PID via home server
+3pd Can bind and unbind 3PID via homeserver
+3pd Can unbind 3PID via homeserver when bound out of band
+3pd 3PIDs are unbound after account deactivation
+3pd Can bind and unbind 3PID via /unbind by specifying the identity server
+3pd Can bind and unbind 3PID via /unbind without specifying the identity server
+app AS can create a user
+app AS can create a user with an underscore
+app AS can create a user with inhibit_login
+app AS cannot create users outside its own namespace
+app Regular users cannot register within the AS namespace
+app AS can make room aliases
+app Regular users cannot create room aliases within the AS namespace
+app AS-ghosted users can use rooms via AS
+app AS-ghosted users can use rooms themselves
+app Ghost user must register before joining room
+app AS can set avatar for ghosted users
+app AS can set displayname for ghosted users
+app AS can't set displayname for random users
+app Inviting an AS-hosted user asks the AS server
+app Accesing an AS-hosted room alias asks the AS server
+app Events in rooms with AS-hosted room aliases are sent to AS server
+app AS user (not ghost) can join room without registering
+app AS user (not ghost) can join room without registering, with user_id query param
+app HS provides query metadata
+app HS can provide query metadata on a single protocol
+app HS will proxy request for 3PU mapping
+app HS will proxy request for 3PL mapping
+app AS can publish rooms in their own list
+app AS and main public room lists are separate
+app AS can deactivate a user
+psh Test that a message is pushed
+psh Invites are pushed
+psh Rooms with names are correctly named in pushed
+psh Rooms with canonical alias are correctly named in pushed
+psh Rooms with many users are correctly pushed
+psh Don't get pushed for rooms you've muted
+psh Rejected events are not pushed
+psh Can add global push rule for room
+psh Can add global push rule for sender
+psh Can add global push rule for content
+psh Can add global push rule for override
+psh Can add global push rule for underride
+psh Can add global push rule for content
+psh New rules appear before old rules by default
+psh Can add global push rule before an existing rule
+psh Can add global push rule after an existing rule
+psh Can delete a push rule
+psh Can disable a push rule
+psh Adding the same push rule twice is idempotent
+psh Messages that notify from another user increment unread notification count
+psh Messages that highlight from another user increment unread highlight count
+psh Can change the actions of default rules
+psh Changing the actions of an unknown default rule fails with 404
+psh Can change the actions of a user specified rule
+psh Changing the actions of an unknown rule fails with 404
+psh Can fetch a user's pushers
+psh Push rules come down in an initial /sync
+psh Adding a push rule wakes up an incremental /sync
+psh Disabling a push rule wakes up an incremental /sync
+psh Enabling a push rule wakes up an incremental /sync
+psh Setting actions for a push rule wakes up an incremental /sync
+psh Can enable/disable default rules
+psh Enabling an unknown default rule fails with 404
+psh Test that rejected pushers are removed.
+psh Notifications can be viewed with GET /notifications
+psh Trying to add push rule with no scope fails with 400
+psh Trying to add push rule with invalid scope fails with 400
+psh Trying to add push rule with missing template fails with 400
+psh Trying to add push rule with missing rule_id fails with 400
+psh Trying to add push rule with empty rule_id fails with 400
+psh Trying to add push rule with invalid template fails with 400
+psh Trying to add push rule with rule_id with slashes fails with 400
+psh Trying to add push rule with override rule without conditions fails with 400
+psh Trying to add push rule with underride rule without conditions fails with 400
+psh Trying to add push rule with condition without kind fails with 400
+psh Trying to add push rule with content rule without pattern fails with 400
+psh Trying to add push rule with no actions fails with 400
+psh Trying to add push rule with invalid action fails with 400
+psh Trying to add push rule with invalid attr fails with 400
+psh Trying to add push rule with invalid value for enabled fails with 400
+psh Trying to get push rules with no trailing slash fails with 400
+psh Trying to get push rules with scope without trailing slash fails with 400
+psh Trying to get push rules with template without tailing slash fails with 400
+psh Trying to get push rules with unknown scope fails with 400
+psh Trying to get push rules with unknown template fails with 400
+psh Trying to get push rules with unknown attribute fails with 400
+psh Trying to get push rules with unknown rule_id fails with 404
+v1s GET /initialSync with non-numeric 'limit'
+v1s GET /events with non-numeric 'limit'
+v1s GET /events with negative 'limit'
+v1s GET /events with non-numeric 'timeout'
+ath Event size limits
+syn Check creating invalid filters returns 4xx
+f,pre New federated private chats get full presence information (SYN-115)
+pre Left room members do not cause problems for presence
+crm Rooms can be created with an initial invite list (SYN-205)
+typ Typing notifications don't leak
+ban Non-present room members cannot ban others
+psh Getting push rules doesn't corrupt the cache SYN-390
+inv Test that we can be reinvited to a room we created
+syn Multiple calls to /sync should not cause 500 errors
+gst Guest user can call /events on another world_readable room (SYN-606)
+gst Real user can call /events on another world_readable room (SYN-606)
+gst Events come down the correct room
+pub Asking for a remote rooms list, but supplying the local server's name, returns the local rooms list
+std Can send a to-device message to two users which both receive it using /sync
diff --git a/tests/sytest/are-we-synapse-yet.py b/tests/sytest/are-we-synapse-yet.py
new file mode 100755
index 0000000..0b334ba
--- /dev/null
+++ b/tests/sytest/are-we-synapse-yet.py
@@ -0,0 +1,260 @@
+#!/usr/bin/env python3
+
+from __future__ import division
+import argparse
+import re
+import sys
+
+# Usage: $ ./are-we-synapse-yet.py [-v] results.tap
+# This script scans a results.tap file from Dendrite's CI process and spits out
+# a rating of how close we are to Synapse parity, based purely on SyTests.
+# The main complexity is grouping tests sensibly into features like 'Registration'
+# and 'Federation'. Then it just checks the ones which are passing and calculates
+# percentages for each group. Produces results like:
+#
+# Client-Server APIs: 29% (196/666 tests)
+# -------------------
+#   Registration             :  62% (20/32 tests)
+#   Login                    :   7% (1/15 tests)
+#   V1 CS APIs               :  10% (3/30 tests)
+#   ...
+#
+# or in verbose mode:
+#
+# Client-Server APIs: 29% (196/666 tests)
+# -------------------
+#  Registration             :  62% (20/32 tests)
+#    ✓ GET /register yields a set of flows
+#    ✓ POST /register can create a user
+#    ✓ POST /register downcases capitals in usernames
+#    ...
+#
+# You can also tack `-v` on to see exactly which tests each category falls under.
+
+test_mappings = {
+    "nsp": "Non-Spec API",
+    "f": "Federation", # flag to mark test involves federation
+
+    "federation_apis": {
+        "fky": "Key API",
+        "fsj": "send_join API",
+        "fmj": "make_join API",
+        "fsl": "send_leave API",
+        "fiv": "Invite API",
+        "fqu": "Query API",
+        "frv": "room versions",
+        "fau": "Auth",
+        "fbk": "Backfill API",
+        "fme": "get_missing_events API",
+        "fst": "State APIs",
+        "fpb": "Public Room API",
+        "fdk": "Device Key APIs",
+        "fed": "Federation API",
+    },
+
+    "client_apis": {
+        "reg": "Registration",
+        "log": "Login",
+        "lox": "Logout",
+        "v1s": "V1 CS APIs",
+        "csa": "Misc CS APIs",
+        "pro": "Profile",
+        "dev": "Devices",
+        "dvk": "Device Keys",
+        "pre": "Presence",
+        "crm": "Create Room",
+        "syn": "Sync API",
+        "rmv": "Room Versions",
+        "rst": "Room State APIs",
+        "pub": "Public Room APIs",
+        "mem": "Room Membership",
+        "ali": "Room Aliases",
+        "jon": "Joining Rooms",
+        "lev": "Leaving Rooms",
+        "inv": "Inviting users to Rooms",
+        "ban": "Banning users",
+        "snd": "Sending events",
+        "get": "Getting events for Rooms",
+        "rct": "Receipts",
+        "red": "Read markers",
+        "med": "Media APIs",
+        "cap": "Capabilities API",
+        "typ": "Typing API",
+        "psh": "Push APIs",
+        "acc": "Account APIs",
+        "eph": "Ephemeral Events",
+        "plv": "Power Levels",
+        "xxx": "Redaction",
+        "3pd": "Third-Party ID APIs",
+        "gst": "Guest APIs",
+        "ath": "Room Auth",
+        "fgt": "Forget APIs",
+        "ctx": "Context APIs",
+        "upg": "Room Upgrade APIs",
+        "tag": "Tagging APIs",
+        "sch": "Search APIs",
+        "oid": "OpenID API",
+        "std": "Send-to-Device APIs",
+        "adm": "Server Admin API",
+        "ign": "Ignore Users",
+        "udr": "User Directory APIs",
+        "app": "Application Services API",
+    },
+}
+
+# optional 'not ' with test number then anything but '#'
+re_testname = re.compile(r"^(not )?ok [0-9]+ ([^#]+)")
+
+# Parses lines like the following:
+#
+# SUCCESS:     ok 3 POST /register downcases capitals in usernames
+# FAIL:        not ok 54 (expected fail) POST /createRoom creates a room with the given version
+# SKIP:        ok 821 Multiple calls to /sync should not cause 500 errors # skip lack of can_post_room_receipts
+# EXPECT FAIL: not ok 822 (expected fail) Guest user can call /events on another world_readable room (SYN-606) # TODO expected fail
+#
+# Only SUCCESS lines are treated as success, the rest are not implemented.
+#
+# Returns a dict like:
+# { name: "...", ok: True }
+def parse_test_line(line):
+    if not line.startswith("ok ") and not line.startswith("not ok "):
+        return
+    re_match = re_testname.match(line)
+    test_name = re_match.groups()[1].replace("(expected fail) ", "").strip()
+    test_pass = False
+    if line.startswith("ok ") and not "# skip " in line:
+        test_pass = True
+    return {
+        "name": test_name,
+        "ok": test_pass,
+    }
+
+# Prints the stats for a complete section.
+#   header_name => "Client-Server APIs"
+#   gid_to_tests => { gid: { <name>: True|False }}
+#   gid_to_name  => { gid: "Group Name" }
+#   verbose => True|False
+# Produces:
+# Client-Server APIs: 29% (196/666 tests)
+# -------------------
+#   Registration             :  62% (20/32 tests)
+#   Login                    :   7% (1/15 tests)
+#   V1 CS APIs               :  10% (3/30 tests)
+#   ...
+# or in verbose mode:
+# Client-Server APIs: 29% (196/666 tests)
+# -------------------
+#  Registration             :  62% (20/32 tests)
+#    ✓ GET /register yields a set of flows
+#    ✓ POST /register can create a user
+#    ✓ POST /register downcases capitals in usernames
+#    ...
+def print_stats(header_name, gid_to_tests, gid_to_name, verbose):
+    subsections = [] # Registration: 100% (13/13 tests)
+    subsection_test_names = {} # 'subsection name': ["✓ Test 1", "✓ Test 2", "× Test 3"]
+    total_passing = 0
+    total_tests = 0
+    for gid, tests in gid_to_tests.items():
+        group_total = len(tests)
+        group_passing = 0
+        test_names_and_marks = []
+        for name, passing in tests.items():
+            if passing:
+                group_passing += 1
+            test_names_and_marks.append(f"{'✓' if passing else '×'} {name}")
+
+        total_tests += group_total
+        total_passing += group_passing
+        pct = "{0:.0f}%".format(group_passing/group_total * 100)
+        line = "%s: %s (%d/%d tests)" % (gid_to_name[gid].ljust(25, ' '), pct.rjust(4, ' '), group_passing, group_total)
+        subsections.append(line)
+        subsection_test_names[line] = test_names_and_marks
+
+    pct = "{0:.0f}%".format(total_passing/total_tests * 100)
+    print("%s: %s (%d/%d tests)" % (header_name, pct, total_passing, total_tests))
+    print("-" * (len(header_name)+1))
+    for line in subsections:
+        print("  %s" % (line,))
+        if verbose:
+            for test_name_and_pass_mark in subsection_test_names[line]:
+                print("    %s" % (test_name_and_pass_mark,))
+            print("")
+    print("")
+
+def main(results_tap_path, verbose):
+    # Load up test mappings
+    test_name_to_group_id = {}
+    fed_tests = set()
+    client_tests = set()
+    groupless_tests = set()
+    with open("./are-we-synapse-yet.list", "r") as f:
+        for line in f.readlines():
+            test_name = " ".join(line.split(" ")[1:]).strip()
+            groups = line.split(" ")[0].split(",")
+            for gid in groups:
+                if gid == "f" or gid in test_mappings["federation_apis"]:
+                    fed_tests.add(test_name)
+                else:
+                    client_tests.add(test_name)
+                if gid == "f":
+                    continue # we expect another group ID
+                test_name_to_group_id[test_name] = gid
+
+    # parse results.tap
+    summary = {
+        "client": {
+            # gid: {
+            #   test_name: OK
+            # }
+        },
+        "federation": {
+            # gid: {
+            #   test_name: OK
+            # }
+        },
+        "nonspec": {
+            "nsp": {}
+        },
+    }
+    with open(results_tap_path, "r") as f:
+        for line in f.readlines():
+            test_result = parse_test_line(line)
+            if not test_result:
+                continue
+            name = test_result["name"]
+            group_id = test_name_to_group_id.get(name)
+            if not group_id:
+                groupless_tests.add(name)
+                # raise Exception("The test '%s' doesn't have a group" % (name,))
+            if group_id == "nsp":
+                summary["nonspec"]["nsp"][name] = test_result["ok"]
+            elif group_id in test_mappings["federation_apis"]:
+                group = summary["federation"].get(group_id, {})
+                group[name] = test_result["ok"]
+                summary["federation"][group_id] = group
+            elif group_id in test_mappings["client_apis"]:
+                group = summary["client"].get(group_id, {})
+                group[name] = test_result["ok"]
+                summary["client"][group_id] = group
+
+    print("Are We Synapse Yet?")
+    print("===================")
+    print("")
+    print_stats("Non-Spec APIs", summary["nonspec"], test_mappings, verbose)
+    print_stats("Client-Server APIs", summary["client"], test_mappings["client_apis"], verbose)
+    print_stats("Federation APIs", summary["federation"], test_mappings["federation_apis"], verbose)
+    if verbose:
+        print("The following tests don't have a group:")
+        for name in groupless_tests:
+            print("  %s" % (name,))
+    else:
+        print("%d tests don't have a group" % len(groupless_tests))
+
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("tap_file", help="path to results.tap")
+    parser.add_argument("-v", action="store_true", help="show individual test names in output")
+    args = parser.parse_args()
+    main(args.tap_file, args.v)
diff --git a/tests/sytest/show-expected-fail-tests.sh b/tests/sytest/show-expected-fail-tests.sh
new file mode 100755
index 0000000..320d4eb
--- /dev/null
+++ b/tests/sytest/show-expected-fail-tests.sh
@@ -0,0 +1,105 @@
+#! /bin/bash
+#
+# Parses a results.tap file from SyTest output and a file containing test names (a test whitelist)
+# and checks whether a test name that exists in the whitelist (that should pass), failed or not.
+#
+# An optional blacklist file can be added, also containing test names, where if a test name is
+# present, the script will not error even if the test is in the whitelist file and failed
+#
+# For each of these files, lines starting with '#' are ignored.
+#
+# Usage ./show-expected-fail-tests.sh results.tap whitelist [blacklist]
+
+results_file=$1
+whitelist_file=$2
+blacklist_file=$3
+
+fail_build=0
+
+if [ $# -lt 2 ]; then
+	echo "Usage: $0 results.tap whitelist [blacklist]"
+	exit 1
+fi
+
+if [ ! -f "$results_file" ]; then
+	echo "ERROR: Specified results file '${results_file}' doesn't exist."
+	fail_build=1
+fi
+
+if [ ! -f "$whitelist_file" ]; then
+	echo "ERROR: Specified test whitelist '${whitelist_file}' doesn't exist."
+	fail_build=1
+fi
+
+blacklisted_tests=()
+
+# Check if a blacklist file was provided
+if [ $# -eq 3 ]; then
+	# Read test blacklist file
+	if [ ! -f "$blacklist_file" ]; then
+		echo "ERROR: Specified test blacklist file '${blacklist_file}' doesn't exist."
+		fail_build=1
+	fi
+
+	# Read each line, ignoring those that start with '#'
+	blacklisted_tests=""
+	search_non_comments=$(grep -v '^#' ${blacklist_file})
+	while read -r line ; do
+		# Record the blacklisted test name
+		blacklisted_tests+=("${line}")
+	done <<< "${search_non_comments}"  # This allows us to edit blacklisted_tests in the while loop
+fi
+
+[ "$fail_build" = 0 ] || exit 1
+
+passed_but_expected_fail=$(grep ' # TODO passed but expected fail' ${results_file} | sed -E 's/^ok [0-9]+ (\(expected fail\) )?//' | sed -E 's/( \([0-9]+ subtests\))? # TODO passed but expected fail$//')
+tests_to_add=""
+already_in_whitelist=""
+
+while read -r test_name; do
+	# Ignore empty lines
+	[ "${test_name}" = "" ] && continue
+
+	grep "^${test_name}" "${whitelist_file}" > /dev/null 2>&1
+	if [ "$?" != "0" ]; then
+		# Check if this test name is blacklisted
+		if printf '%s\n' "${blacklisted_tests[@]}" | grep -q -P "^${test_name}$"; then
+			# Don't notify about this test
+			continue
+		fi
+
+		# Append this test_name to the existing list
+		tests_to_add="${tests_to_add}${test_name}\n"
+		fail_build=1
+	else
+		already_in_whitelist="${already_in_whitelist}${test_name}\n"
+	fi
+done <<< "${passed_but_expected_fail}"
+
+# TODO: Check that the same test doesn't exist in both the whitelist and blacklist
+# TODO: Check that the same test doesn't appear twice in the whitelist|blacklist
+
+# Trim test output strings
+tests_to_add=$(IFS=$'\n' echo "${tests_to_add[*]%%'\n'}")
+already_in_whitelist=$(IFS=$'\n' echo "${already_in_whitelist[*]%%'\n'}")
+
+# Format output with markdown for buildkite annotation rendering purposes
+if [ -n "${tests_to_add}" ] && [ -n "${already_in_whitelist}" ]; then
+	echo "### 📜 SyTest Whitelist Maintenance"
+fi
+
+if [ -n "${tests_to_add}" ]; then
+	echo "**ERROR**: The following tests passed but are not present in \`$2\`. Please append them to the file:"
+    echo "\`\`\`"
+    echo -e "${tests_to_add}"
+    echo "\`\`\`"
+fi
+
+if [ -n "${already_in_whitelist}" ]; then
+	echo "**WARN**: Tests in the whitelist still marked as **expected fail**:"
+    echo "\`\`\`"
+    echo -e "${already_in_whitelist}"
+    echo "\`\`\`"
+fi
+
+exit ${fail_build}
diff --git a/tests/sytest/sytest-blacklist b/tests/sytest/sytest-blacklist
new file mode 100644
index 0000000..009de22
--- /dev/null
+++ b/tests/sytest/sytest-blacklist
@@ -0,0 +1,7 @@
+# This test checks for a room-alias key in the response which is not in the spec, we must add it back in whitelist when https://github.com/matrix-org/sytest/pull/880 is merged
+POST /createRoom makes a public room
+# These fails because they use a endpoint which is not in the spec, we must add them back in whitelist when https://github.com/matrix-org/sytest/issues/878 is closed
+POST /createRoom makes a room with a name
+POST /createRoom makes a room with a topic
+Can /sync newly created room
+POST /createRoom ignores attempts to set the room version via creation_content
\ No newline at end of file
diff --git a/tests/sytest/sytest-whitelist b/tests/sytest/sytest-whitelist
new file mode 100644
index 0000000..e1f4e5c
--- /dev/null
+++ b/tests/sytest/sytest-whitelist
@@ -0,0 +1,126 @@
+/joined_members return joined members
+/joined_rooms returns only joined rooms
+3pid invite join valid signature but revoked keys are rejected
+3pid invite join valid signature but unreachable ID server are rejected
+3pid invite join with wrong but valid signature are rejected
+AS cannot create users outside its own namespace
+After deactivating account, can't log in with an email
+Alias creators can delete alias with no ops
+Alias creators can delete canonical alias with no ops
+Alternative server names do not cause a routing loop
+Both GET and PUT work
+Can add account data
+Can add tag
+Can create filter
+Can delete canonical alias
+Can invite users to invite-only rooms
+Can list tags for a room
+Can logout all devices
+Can logout current device
+Can re-join room if re-invited
+Can read configuration endpoint
+Can recv a device message using /sync
+Can recv device messages until they are acknowledged
+Can remove tag
+Can send a message directly to a device using PUT /sendToDevice
+Can send a to-device message to two users which both receive it using /sync
+Can send messages with a wildcard device id
+Can send messages with a wildcard device id to two devices
+Can sync
+Can upload with ASCII file name
+Can upload with Unicode file name
+Can upload without a file name
+Changing the actions of an unknown default rule fails with 404
+Changing the actions of an unknown rule fails with 404
+Checking local federation server
+Current state appears in timeline in private history
+Current state appears in timeline in private history with many messages before
+Deleted tags appear in an incremental v2 /sync
+Deleting a non-existent alias should return a 404
+Device messages wake up /sync
+Device messages with the same txn_id are deduplicated
+Events come down the correct room
+GET /device/{deviceId}
+GET /device/{deviceId} gives a 404 for unknown devices
+GET /devices
+GET /events with negative 'limit'
+GET /events with non-numeric 'limit'
+GET /events with non-numeric 'timeout'
+GET /joined_rooms lists newly-created room
+GET /login yields a set of flows
+GET /media/r0/download can fetch the value again
+GET /profile/:user_id/avatar_url publicly accessible
+GET /profile/:user_id/displayname publicly accessible
+GET /publicRooms lists newly-created room
+GET /register yields a set of flows
+GET /rooms/:room_id/state fetches entire room state
+GET /rooms/:room_id/state/m.room.member/:user_id fetches my membership
+Getting push rules doesn't corrupt the cache SYN-390
+Newly updated tags appear in an incremental v2 /sync
+POST /createRoom makes a private room
+POST /createRoom makes a private room with invites
+POST /join/:room_id can join a room
+POST /login as non-existing user is rejected
+POST /login can log in as a user
+POST /login can log in as a user with just the local part of the id
+POST /login returns the same device_id as that in the request
+POST /login wrong password is rejected
+POST /media/r0/upload can create an upload
+POST /register can create a user
+POST /register downcases capitals in usernames
+POST /register rejects registration of usernames with '!'
+POST /register rejects registration of usernames with '"'
+POST /register rejects registration of usernames with '''
+POST /register rejects registration of usernames with ':'
+POST /register rejects registration of usernames with '?'
+POST /register rejects registration of usernames with '@'
+POST /register rejects registration of usernames with '['
+POST /register rejects registration of usernames with '\'
+POST /register rejects registration of usernames with '\n'
+POST /register rejects registration of usernames with ']'
+POST /register rejects registration of usernames with '{'
+POST /register rejects registration of usernames with '|'
+POST /register rejects registration of usernames with '}'
+POST /register rejects registration of usernames with '£'
+POST /register rejects registration of usernames with 'é'
+POST /rooms/:room_id/ban can ban a user
+POST /rooms/:room_id/invite can send an invite
+POST /rooms/:room_id/join can join a room
+POST /rooms/:room_id/leave can leave a room
+POST /rooms/:room_id/state/m.room.name sets name
+POST /rooms/:room_id/state/m.room.topic sets topic
+POST /rooms/:room_id/upgrade can upgrade a room version
+POSTed media can be thumbnailed
+PUT /device/{deviceId} gives a 404 for unknown devices
+PUT /device/{deviceId} updates device fields
+PUT /directory/room/:room_alias creates alias
+PUT /profile/:user_id/avatar_url sets my avatar
+PUT /profile/:user_id/displayname sets my name
+PUT /rooms/:room_id/state/m.room.power_levels can set levels
+PUT power_levels should not explode if the old power levels were empty
+Push rules come down in an initial /sync
+Read markers appear in incremental v2 /sync
+Read markers appear in initial v2 /sync
+Read markers can be updated
+Regular users can add and delete aliases in the default room configuration
+Room creation reports m.room.create to myself
+Room creation reports m.room.member to myself
+Rooms a user is invited to appear in an incremental sync
+Rooms a user is invited to appear in an initial sync
+Setting room topic reports m.room.topic to myself
+Should reject keys claiming to belong to a different user
+Tags appear in an initial v2 /sync
+Trying to get push rules with unknown rule_id fails with 404
+Typing events appear in gapped sync
+Typing events appear in incremental sync
+Typing events appear in initial sync
+Uninvited users cannot join the room
+User appears in user directory
+User in dir while user still shares private rooms
+User in shared private room does appear in user directory
+User is offline if they set_presence=offline in their sync
+Users with sufficient power-level can delete other's aliases
+Version responds 200 OK with valid structure
+We should see our own leave event when rejecting an invite, even if history_visibility is restricted (riot-web/3462)
+Wildcard device messages wake up /sync
+query for user with no keys returns empty key dict