Don't listen on :8448 for the Complement federation server (#289)

* Don't listen on :8448 for the Complement federation server

Instead, listen on a random OS-allocated high numbered port then
do a switcheroo on the `ServerName` so it reads correctly e.g
`host.docker.internal:56185`. This means the server name will be
invalid if it is read before `Server.Listen()` is called so we now
guard common access points in `Server` which rely on the server
name and fail tests if the server is not yet listening when those
functions are called. To further guard against misuse of server
name whilst it isn't valid, turn it into a private field.

* Bind to localhost

* Listen on all interfaces again as we need it to listen on Docker interfaces in CI

* Experimental support for Complement on ubuntu VM in GHA

* Tweak

* More tweaking

* More tweaks

* More

* Guessing at this point

* Maybe

* Modify GOROOT

* Set go version before running other complement install commands

* Docstrings and BUILDKIT=1

* No need to cd complement-master

* Add GOPATH

* Install libolm-dev; more docs

* Say we're not running under CI

* Remove CI flag

We don't need it anymore as CI does not run Complement inside Docker

* Shadow lint

* Fix shadowing

.github/workflows/ci.yaml

@@ -31,18 +31,26 @@ jobs:
             tags: msc2836 dendrite_blacklist
             default_branch: master
 
-    container:
-      image: matrixdotorg/complement  # dockerfiles/ComplementCIBuildkite.Dockerfile
-      env:
-        CI: true
-        DOCKER_BUILDKIT: 1
-      ports:
-        - 8448:8448
-      volumes:
-        - /var/run/docker.sock:/var/run/docker.sock
-
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v2 # Checkout complement
+
+      # Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env.
+      # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
+      - name: "Set Go Version"
+        run: |
+          echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
+          echo "~/go/bin" >> $GITHUB_PATH
+
+      # Similar steps as dockerfiles/ComplementCIBuildkite.Dockerfile but on the host. We need
+      # to do this so we can _be_ the host when running Complement so we can snaffle all the ports. If
+      # we run Complement _in_ Docker then we can't -p all high numbered ports which then breaks federation
+      # servers which listen on random high numbered ports.
+      - name: "Install Complement Dependencies"
+        # We don't need to install Go because it is included on the Ubuntu 20.04 image:
+        # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64
+        run: |
+          sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
+          go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
 
       - name: "Checkout corresponding ${{ matrix.homeserver }} branch"
         # This is only done for Synapse since Dendrite's docker file pulls in
@@ -80,9 +88,15 @@ jobs:
         # built docker image).
         if: ${{ matrix.homeserver == 'Synapse' }}
         working-directory: homeserver
+        env:
+          DOCKER_BUILDKIT: 1
 
       - run: docker build -t homeserver -f dockerfiles/${{ matrix.homeserver }}.Dockerfile dockerfiles/
-      - run: set -o pipefail && go test -p 2 -v -json -tags "${{ matrix.tags }}" ./tests/... 2>&1 | gotestfmt
+      - run: |
+          set -o pipefail &&
+          go test -p 2 -v -json -tags "${{ matrix.tags }}" ./tests/... 2>&1 | gotestfmt
         shell: bash # required for pipefail to be A Thing. pipefail is required to stop gotestfmt swallowing non-zero exit codes
+        name: Run Complement Tests
         env:
           COMPLEMENT_BASE_IMAGE: homeserver
+          DOCKER_BUILDKIT: 1

internal/docker/builder.go

@@ -17,7 +17,6 @@ import (
 	"context"
 	"fmt"
 	"log"
-	"os"
 	"strings"
 	"time"
 
@@ -39,15 +38,6 @@ var (
 	HostnameRunningDocker = "localhost"
 )
 
-func init() {
-	if os.Getenv("CI") == "true" {
-		log.Println("Running under CI: redirecting localhost to docker host on 172.17.0.1")
-		// this assumes we are running inside docker so they have
-		// forwarded the docker socket to us and we're in a container.
-		HostnameRunningDocker = "172.17.0.1"
-	}
-}
-
 const complementLabel = "complement_context"
 
 type Builder struct {

internal/docker/volumes.go

@@ -1,13 +1,9 @@
 package docker
 
 import (
-	"bufio"
 	"context"
-	"fmt"
-	"io/ioutil"
 	"os"
 	"path"
-	"strings"
 
 	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/api/types/volume"
@@ -31,50 +27,21 @@ type VolumeCA struct {
 // Prepare the Certificate Authority volume. This is independent of the homeserver calling Prepare
 // hence the contextual string is unused.
 func (v *VolumeCA) Prepare(ctx context.Context, docker *client.Client, x string) error {
-	// TODO: wrap in a lockfile
-	if os.Getenv("CI") == "true" {
-		// When in CI, Complement itself is a container with the CA volume mounted at /ca.
-		// We need to mount this volume to all homeserver containers to synchronize the CA cert.
-		// This is needed to establish trust among all containers.
-
-		containerID := getContainerID()
-		if containerID == "" {
-			return fmt.Errorf("failed to get container ID")
-		}
-		container, err := docker.ContainerInspect(ctx, containerID)
-		if err != nil {
-			return err
-		}
-		// Get the volume that matches the destination in our complement container
-		for i := range container.Mounts {
-			if container.Mounts[i].Destination == "/ca" {
-				v.source = container.Mounts[i].Name
-				v.typ = container.Mounts[i].Type
-				break
-			}
-		}
-		if v.source == "" {
-			// We did not find a volume. This container might be created without a volume,
-			// or CI=true is passed but we are not running in a container.
-			return fmt.Errorf("CI=true but there is no /ca mounted to Complement's container")
-		}
-	} else {
-		// When not in CI, our CA cert is placed in the current working dir.
-		// We bind mount this directory to all homeserver containers.
-		cwd, err := os.Getwd()
+	// Our CA cert is placed in the current working dir.
+	// We bind mount this directory to all homeserver containers.
+	cwd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+	caCertificateDirHost := path.Join(cwd, "ca")
+	if _, err := os.Stat(caCertificateDirHost); os.IsNotExist(err) {
+		err = os.Mkdir(caCertificateDirHost, 0770)
 		if err != nil {
 			return err
 		}
-		caCertificateDirHost := path.Join(cwd, "ca")
-		if _, err := os.Stat(caCertificateDirHost); os.IsNotExist(err) {
-			err = os.Mkdir(caCertificateDirHost, 0770)
-			if err != nil {
-				return err
-			}
-		}
-		v.source = path.Join(cwd, "ca")
-		v.typ = mount.TypeBind
 	}
+	v.source = path.Join(cwd, "ca")
+	v.typ = mount.TypeBind
 	return nil
 }
 
@@ -108,60 +75,3 @@ func (v *VolumeAppService) Mount() mount.Mount {
 		Target: "/appservices",
 	}
 }
-
-func getContainerID() string {
-	cid, err := getContainerIDViaCPUSet()
-	if err == nil {
-		return cid
-	}
-	fmt.Printf("failed to get container ID via cpuset, trying alternatives: %s\n", err)
-
-	cid, err = getContainerIDViaCGroups()
-	if err == nil {
-		return cid
-	}
-
-	fmt.Printf("failed to get container ID via cgroups, out of options: %s\n", err)
-	return ""
-}
-
-func getContainerIDViaCGroups() (string, error) {
-	file, err := os.Open("/proc/self/cgroup")
-	if err != nil {
-		return "", err
-	}
-
-	scanner := bufio.NewScanner(file)
-	defer file.Close()
-
-	scanner.Split(bufio.ScanLines)
-	for scanner.Scan() {
-		// Returns entries like this on github actions
-		// 9:memory:/actions_job/c8d555525bad6cd896c5aa985ef68010be47b1fb321c95547761c8f1a053b86e
-		line := scanner.Text()
-		segments := strings.Split(line, "/")
-		containerID := segments[len(segments)-1]
-		if containerID == "" || len(containerID) < 64 {
-			continue
-		}
-		return containerID, nil
-	}
-	return "", fmt.Errorf("faild to find container id in cgroups")
-}
-
-func getContainerIDViaCPUSet() (string, error) {
-	// /proc/1/cpuset should be /docker/<containerID>
-	cpuset, err := ioutil.ReadFile("/proc/1/cpuset")
-	if err != nil {
-		return "", err
-	}
-	if !strings.Contains(string(cpuset), "docker") {
-		return "", fmt.Errorf("could not identify container ID using /proc/1/cpuset - cpuset=%s", string(cpuset))
-	}
-	cpusetList := strings.Split(strings.TrimSpace(string(cpuset)), "/")
-	containerID := cpusetList[len(cpusetList)-1]
-	if len(containerID) == 0 {
-		return "", fmt.Errorf("cpuset missing container ID")
-	}
-	return containerID, nil
-}

internal/federation/handle.go

@@ -18,7 +18,7 @@ import (
 func MakeJoinRequestsHandler(s *Server, w http.ResponseWriter, req *http.Request) {
 	// Check federation signature
 	fedReq, errResp := gomatrixserverlib.VerifyHTTPRequest(
-		req, time.Now(), gomatrixserverlib.ServerName(s.ServerName), s.keyRing,
+		req, time.Now(), gomatrixserverlib.ServerName(s.serverName), s.keyRing,
 	)
 	if fedReq == nil {
 		w.WriteHeader(errResp.Code)
@@ -74,7 +74,7 @@ func MakeJoinRequestsHandler(s *Server, w http.ResponseWriter, req *http.Request
 // HandleMakeSendJoinRequests.
 func SendJoinRequestsHandler(s *Server, w http.ResponseWriter, req *http.Request) {
 	fedReq, errResp := gomatrixserverlib.VerifyHTTPRequest(
-		req, time.Now(), gomatrixserverlib.ServerName(s.ServerName), s.keyRing,
+		req, time.Now(), gomatrixserverlib.ServerName(s.serverName), s.keyRing,
 	)
 	if fedReq == nil {
 		w.WriteHeader(errResp.Code)
@@ -106,9 +106,9 @@ func SendJoinRequestsHandler(s *Server, w http.ResponseWriter, req *http.Request
 
 	// return state and auth chain
 	b, err := json.Marshal(gomatrixserverlib.RespSendJoin{
+		Origin:      gomatrixserverlib.ServerName(s.serverName),
 		AuthEvents:  authEvents,
 		StateEvents: stateEvents,
-		Origin:      gomatrixserverlib.ServerName(s.ServerName),
 	})
 	if err != nil {
 		w.WriteHeader(500)
@@ -141,7 +141,7 @@ func HandleInviteRequests(inviteCallback func(*gomatrixserverlib.Event)) func(*S
 		// https://matrix.org/docs/spec/server_server/r0.1.4#put-matrix-federation-v2-invite-roomid-eventid
 		s.mux.Handle("/_matrix/federation/v2/invite/{roomID}/{eventID}", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 			fedReq, errResp := gomatrixserverlib.VerifyHTTPRequest(
-				req, time.Now(), gomatrixserverlib.ServerName(s.ServerName), s.keyRing,
+				req, time.Now(), gomatrixserverlib.ServerName(s.serverName), s.keyRing,
 			)
 			if fedReq == nil {
 				w.WriteHeader(errResp.Code)
@@ -169,7 +169,7 @@ func HandleInviteRequests(inviteCallback func(*gomatrixserverlib.Event)) func(*S
 			}
 
 			// Sign the event before we send it back
-			signedEvent := inviteRequest.Event().Sign(s.ServerName, s.KeyID, s.Priv)
+			signedEvent := inviteRequest.Event().Sign(s.serverName, s.KeyID, s.Priv)
 
 			// Send the response
 			res := map[string]interface{}{
@@ -195,7 +195,7 @@ func HandleDirectoryLookups() func(*Server) {
 				b, err := json.Marshal(gomatrixserverlib.RespDirectory{
 					RoomID: roomID,
 					Servers: []gomatrixserverlib.ServerName{
-						gomatrixserverlib.ServerName(s.ServerName),
+						gomatrixserverlib.ServerName(s.serverName),
 					},
 				})
 				if err != nil {
@@ -235,7 +235,7 @@ func HandleEventRequests() func(*Server) {
 			}
 
 			txn := gomatrixserverlib.Transaction{
-				Origin:         gomatrixserverlib.ServerName(srv.ServerName),
+				Origin:         gomatrixserverlib.ServerName(srv.serverName),
 				OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()),
 				PDUs: []json.RawMessage{
 					event.JSON(),
@@ -259,7 +259,7 @@ func HandleKeyRequests() func(*Server) {
 		keymux := srv.mux.PathPrefix("/_matrix/key/v2").Subrouter()
 		keyFn := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 			k := gomatrixserverlib.ServerKeys{}
-			k.ServerName = gomatrixserverlib.ServerName(srv.ServerName)
+			k.ServerName = gomatrixserverlib.ServerName(srv.serverName)
 			publicKey := srv.Priv.Public().(ed25519.PublicKey)
 			k.VerifyKeys = map[gomatrixserverlib.KeyID]gomatrixserverlib.VerifyKey{
 				srv.KeyID: {
@@ -276,7 +276,7 @@ func HandleKeyRequests() func(*Server) {
 			}
 
 			k.Raw, err = gomatrixserverlib.SignJSON(
-				string(srv.ServerName), srv.KeyID, srv.Priv, toSign,
+				string(srv.serverName), srv.KeyID, srv.Priv, toSign,
 			)
 			if err != nil {
 				w.WriteHeader(500)
@@ -304,9 +304,9 @@ func HandleMediaRequests(mediaIds map[string]func(w http.ResponseWriter)) func(*
 			origin := vars["origin"]
 			mediaId := vars["mediaId"]
 
-			if origin != srv.ServerName {
+			if origin != srv.serverName {
 				w.WriteHeader(400)
-				w.Write([]byte("complement: Invalid Origin; Expected " + srv.ServerName))
+				w.Write([]byte("complement: Invalid Origin; Expected " + srv.serverName))
 				return
 			}
 
@@ -338,7 +338,7 @@ func HandleTransactionRequests(pduCallback func(*gomatrixserverlib.Event), eduCa
 
 			// Check federation signature
 			fedReq, errResp := gomatrixserverlib.VerifyHTTPRequest(
-				req, time.Now(), gomatrixserverlib.ServerName(srv.ServerName), srv.keyRing,
+				req, time.Now(), gomatrixserverlib.ServerName(srv.serverName), srv.keyRing,
 			)
 			if fedReq == nil {
 				log.Printf(