Merge branch 'matrix-org:main' into main

.github/ISSUE_TEMPLATE/BUG_REPORT.md

@@ -62,6 +62,6 @@ If you can identify any relevant log snippets from server logs, please include
 those (please be careful to remove any personal or private data). Please surround them with
 ``` (three backticks, on a line on their own), so that they are formatted legibly.
 
-Alternatively, please send logs to @kegan:matrix.org or @neilalexander:matrix.org
+Alternatively, please send logs to @kegan:matrix.org, @s7evink:matrix.org or @devonh:one.ems.host
 with a link to the respective Github issue, thanks!
 -->

.github/workflows/dendrite.yml

@@ -440,7 +440,7 @@ jobs:
       # Run Complement
       - run: |
           set -o pipefail &&
-          go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt -hide all
+          go test -v -json -tags dendrite_blacklist ./tests ./tests/csapi 2>&1 | gotestfmt -hide all
         shell: bash
         name: Run Complement Tests
         env:

.github/workflows/docker.yml

@@ -32,10 +32,6 @@ jobs:
         if: github.event_name == 'release' # Only for GitHub releases
         run: |
           echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
@@ -60,7 +56,6 @@ jobs:
           cache-from: type=registry,ref=ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:buildcache
           cache-to: type=registry,ref=ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:buildcache,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -75,7 +70,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -109,10 +103,6 @@ jobs:
         if: github.event_name == 'release' # Only for GitHub releases
         run: |
           echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
@@ -137,7 +127,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           file: ./build/docker/Dockerfile.demo-pinecone
           platforms: ${{ env.PLATFORMS }}
           push: true
@@ -153,7 +142,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           file: ./build/docker/Dockerfile.demo-pinecone
           platforms: ${{ env.PLATFORMS }}
           push: true
@@ -176,10 +164,6 @@ jobs:
         if: github.event_name == 'release' # Only for GitHub releases
         run: |
           echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
@@ -204,7 +188,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           file: ./build/docker/Dockerfile.demo-yggdrasil
           platforms: ${{ env.PLATFORMS }}
           push: true
@@ -220,7 +203,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           file: ./build/docker/Dockerfile.demo-yggdrasil
           platforms: ${{ env.PLATFORMS }}
           push: true

.github/workflows/helm.yml

@@ -32,7 +32,7 @@ jobs:
           version: v3.10.0
 
       - name: Run chart-releaser
-        uses: helm/chart-releaser-action@v1.4.1
+        uses: helm/chart-releaser-action@ed43eb303604cbc0eeec8390544f7748dc6c790d # specific commit, since `mark_as_latest` is not yet in a release
         env:
           CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
         with:

.golangci.yml

@@ -180,7 +180,6 @@ linters-settings:
 linters:
   enable:
     - errcheck
-    - goconst
     - gocyclo
     - goimports # Does everything gofmt does
     - gosimple
@@ -211,6 +210,7 @@ linters:
     - stylecheck
     - typecheck # Should turn back on soon
     - unconvert # Should turn back on soon
+    - goconst # Slightly annoying, as it reports "issues" in SQL statements
   disable-all: false
   presets:
   fast: false

CHANGES.md

@@ -1,5 +1,42 @@
 # Changelog
 
+## Dendrite 0.13.4 (2023-10-25)
+
+Upgrading to this version is **highly** recommended, as it fixes a long-standing bug in the state resolution
+algorithm.
+
+### Fixes:
+
+- The "device list updater" now de-duplicates the servers to fetch devices from on startup. (This also 
+  avoids spamming the logs when shutting down.)
+- A bug in the state resolution algorithm has been fixed. This bug could result in users "being reset"
+  out of rooms and other missing state events due to calculating the wrong state.
+- A bug when setting notifications from Element Android has been fixed by implementing MSC3987
+
+### Features
+
+- Updated dependencies
+  - Internal NATS Server has been updated from v2.9.19 to v2.9.23
+
+## Dendrite 0.13.3 (2023-09-28)
+
+### Fixes:
+
+- The `user_id` query parameter when authenticating is now used correctly (contributed by [tulir](https://github.com/tulir))
+- Invitations are now correctly pushed to devices
+- A bug which could result in the corruption of `m.direct` account data has been fixed 
+
+### Features
+
+- [Sliding Sync proxy](https://github.com/matrix-org/sliding-sync) can be configured in the `/.well-known/matrix/client` response
+- Room version 11 is now supported
+- Clients can request the `federation` `event_format` when creating filters
+- Many under the hood improvements for [MSC4014: Pseudonymous Identities](https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/pseudo-ids/proposals/4014-pseudonymous-identities.md)
+
+### Other
+
+- Dendrite now requires Go 1.20 if building from source
+
 ## Dendrite 0.13.2 (2023-08-23)
 
 ### Fixes:

Dockerfile

@@ -3,7 +3,7 @@
 #
 # base installs required dependencies and runs go mod download to cache dependencies
 #
-FROM --platform=${BUILDPLATFORM} docker.io/golang:1.20-alpine AS base
+FROM --platform=${BUILDPLATFORM} docker.io/golang:1.21-alpine AS base
 RUN apk --update --no-cache add bash build-base curl git
 
 #
@@ -13,15 +13,14 @@ FROM --platform=${BUILDPLATFORM} base AS build
 WORKDIR /src
 ARG TARGETOS
 ARG TARGETARCH
-ARG FLAGS
 RUN --mount=target=. \
     --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=cache,target=/go/pkg/mod \
     USERARCH=`go env GOARCH` \
     GOARCH="$TARGETARCH" \
     GOOS="linux" \
     CGO_ENABLED=$([ "$TARGETARCH" = "$USERARCH" ] && echo "1" || echo "0") \
-    go build -v -ldflags="${FLAGS}" -trimpath -o /out/ ./cmd/...
+    go build -v -trimpath -o /out/ ./cmd/...
 
 
 #

README.md

@@ -36,7 +36,7 @@ If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or j
 See the [Planning your Installation](https://matrix-org.github.io/dendrite/installation/planning) page for
 more information on requirements.
 
-To build Dendrite, you will need Go 1.18 or later.
+To build Dendrite, you will need Go 1.20 or later.
 
 For a usable federating Dendrite deployment, you will also need:
 

appservice/consumers/roomserver.go

@@ -128,7 +128,7 @@ func (s *OutputRoomEventConsumer) onMessage(
 			if len(output.NewRoomEvent.AddsStateEventIDs) > 0 {
 				newEventID := output.NewRoomEvent.Event.EventID()
 				eventsReq := &api.QueryEventsByIDRequest{
-					RoomID:   output.NewRoomEvent.Event.RoomID(),
+					RoomID:   output.NewRoomEvent.Event.RoomID().String(),
 					EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)),
 				}
 				eventsRes := &api.QueryEventsByIDResponse{}
@@ -236,11 +236,7 @@ func (s *appserviceState) backoffAndPause(err error) error {
 // TODO: This should be cached, see https://github.com/matrix-org/dendrite/issues/1682
 func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Context, event *types.HeaderedEvent, appservice *config.ApplicationService) bool {
 	user := ""
-	validRoomID, err := spec.NewRoomID(event.RoomID())
-	if err != nil {
-		return false
-	}
-	userID, err := s.rsAPI.QueryUserIDForSender(ctx, *validRoomID, event.SenderID())
+	userID, err := s.rsAPI.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID())
 	if err == nil {
 		user = userID.String()
 	}
@@ -250,7 +246,7 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont
 		return false
 	case appservice.IsInterestedInUserID(user):
 		return true
-	case appservice.IsInterestedInRoomID(event.RoomID()):
+	case appservice.IsInterestedInRoomID(event.RoomID().String()):
 		return true
 	}
 
@@ -261,7 +257,7 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont
 	}
 
 	// Check all known room aliases of the room the event came from
-	queryReq := api.GetAliasesForRoomIDRequest{RoomID: event.RoomID()}
+	queryReq := api.GetAliasesForRoomIDRequest{RoomID: event.RoomID().String()}
 	var queryRes api.GetAliasesForRoomIDResponse
 	if err := s.rsAPI.GetAliasesForRoomID(ctx, &queryReq, &queryRes); err == nil {
 		for _, alias := range queryRes.Aliases {
@@ -272,7 +268,7 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont
 	} else {
 		log.WithFields(log.Fields{
 			"appservice": appservice.ID,
-			"room_id":    event.RoomID(),
+			"room_id":    event.RoomID().String(),
 		}).WithError(err).Errorf("Unable to get aliases for room")
 	}
 
@@ -288,7 +284,7 @@ func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, e
 	// until we have a lighter way of checking the state before the event that
 	// doesn't involve state res, then this is probably OK.
 	membershipReq := &api.QueryMembershipsForRoomRequest{
-		RoomID:     event.RoomID(),
+		RoomID:     event.RoomID().String(),
 		JoinedOnly: true,
 	}
 	membershipRes := &api.QueryMembershipsForRoomResponse{}
@@ -317,7 +313,7 @@ func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, e
 	} else {
 		log.WithFields(log.Fields{
 			"appservice": appservice.ID,
-			"room_id":    event.RoomID(),
+			"room_id":    event.RoomID().String(),
 		}).WithError(err).Errorf("Unable to get membership for room")
 	}
 	return false

are-we-synapse-yet.list

@@ -944,4 +944,12 @@ rmv remote user can join room with version 10
 rmv User can invite remote user to room with version 10
 rmv Remote user can backfill in a room with version 10
 rmv Can reject invites over federation for rooms with version 10
-rmv Can receive redactions from regular users over federation in room version 10
+rmv Can receive redactions from regular users over federation in room version 10
+rmv User can create and send/receive messages in a room with version 11
+rmv local user can join room with version 11
+rmv User can invite local user to room with version 11
+rmv remote user can join room with version 11
+rmv User can invite remote user to room with version 11
+rmv Remote user can backfill in a room with version 11
+rmv Can reject invites over federation for rooms with version 11
+rmv Can receive redactions from regular users over federation in room version 11

build/docker/Dockerfile.demo-yggdrasil

@@ -1,4 +1,4 @@
-FROM docker.io/golang:1.19-alpine AS base
+FROM docker.io/golang:1.21-alpine AS base
 
 #
 # Needs to be separate from the main Dockerfile for OpenShift,

build/scripts/complement.sh

@@ -15,5 +15,5 @@ tar -xzf master.tar.gz
 
 # Run the tests!
 cd complement-master
-COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -v -count=1 ./tests
+COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -v -count=1 ./tests ./tests/csapi
 

clientapi/clientapi_test.go

@@ -1418,7 +1418,7 @@ func TestPushRules(t *testing.T) {
 				validateFunc: func(t *testing.T, respBody *bytes.Buffer) {
 					actions := gjson.GetBytes(respBody.Bytes(), "actions").Array()
 					// only a basic check
-					assert.Equal(t, 1, len(actions))
+					assert.Equal(t, 0, len(actions))
 				},
 			},
 			{

clientapi/routing/profile.go

@@ -300,7 +300,7 @@ func updateProfile(
 		}, e
 	}
 
-	if err := api.SendEvents(ctx, rsAPI, api.KindNew, events, device.UserDomain(), domain, domain, nil, true); err != nil {
+	if err := api.SendEvents(ctx, rsAPI, api.KindNew, events, device.UserDomain(), domain, domain, nil, false); err != nil {
 		util.GetLogger(ctx).WithError(err).Error("SendEvents failed")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,

clientapi/routing/redaction.go

@@ -34,7 +34,8 @@ import (
 )
 
 type redactionContent struct {
-	Reason string `json:"reason"`
+	Reason  string `json:"reason"`
+	Redacts string `json:"redacts"`
 }
 
 type redactionResponse struct {
@@ -98,7 +99,7 @@ func SendRedaction(
 			JSON: spec.NotFound("unknown event ID"), // TODO: is it ok to leak existence?
 		}
 	}
-	if ev.RoomID() != roomID {
+	if ev.RoomID().String() != roomID {
 		return util.JSONResponse{
 			Code: 400,
 			JSON: spec.NotFound("cannot redact event in another room"),
@@ -151,6 +152,11 @@ func SendRedaction(
 		Type:     spec.MRoomRedaction,
 		Redacts:  eventID,
 	}
+
+	// Room version 11 expects the "redacts" field on the
+	// content field, so add it here as well
+	r.Redacts = eventID
+
 	err = proto.SetContent(r)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("proto.SetContent failed")

clientapi/routing/register.go

@@ -236,7 +236,7 @@ type authDict struct {
 	// TODO: Lots of custom keys depending on the type
 }
 
-// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#user-interactive-authentication-api
+// https://spec.matrix.org/v1.7/client-server-api/#user-interactive-authentication-api
 type userInteractiveResponse struct {
 	Flows     []authtypes.Flow       `json:"flows"`
 	Completed []authtypes.LoginType  `json:"completed"`
@@ -256,7 +256,7 @@ func newUserInteractiveResponse(
 	}
 }
 
-// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register
+// https://spec.matrix.org/v1.7/client-server-api/#post_matrixclientv3register
 type registerResponse struct {
 	UserID      string `json:"user_id"`
 	AccessToken string `json:"access_token,omitempty"`
@@ -462,7 +462,7 @@ func validateApplicationService(
 }
 
 // Register processes a /register request.
-// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register
+// https://spec.matrix.org/v1.7/client-server-api/#post_matrixclientv3register
 func Register(
 	req *http.Request,
 	userAPI userapi.ClientUserAPI,