fix(buildtool): let libevent use OpenSSL's pkgconfig

go.mod

@@ -145,9 +145,9 @@ require (
 	github.com/xtaci/kcp-go/v5 v5.6.2 // indirect
 	github.com/xtaci/smux v1.5.24 // indirect
 	gitlab.com/yawning/bsaes.git v0.0.0-20190805113838-0a714cd429ec // indirect
-	golang.org/x/mod v0.12.0 // indirect
+	golang.org/x/mod v0.13.0 // indirect
 	golang.org/x/term v0.13.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
-	golang.org/x/tools v0.13.0 // indirect
+	golang.org/x/tools v0.14.0 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 )

go.sum

@@ -588,8 +588,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
-golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
+golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -626,7 +626,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
+golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -718,8 +718,8 @@ golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ=
-golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
+golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=
+golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
 golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

internal/cmd/buildtool/android_test.go

@@ -754,12 +754,6 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) {
 				"DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm",
 				"install_dev",
 			},
-		}, {
-			Env: []string{},
-			Argv: []string{
-				"rm", "-rf",
-				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm/lib/pkgconfig",
-			},
 		}, {
 			Env: []string{},
 			Argv: []string{
@@ -815,12 +809,6 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) {
 				"DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64",
 				"install_dev",
 			},
-		}, {
-			Env: []string{},
-			Argv: []string{
-				"rm", "-rf",
-				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64/lib/pkgconfig",
-			},
 		}, {
 			Env: []string{},
 			Argv: []string{
@@ -876,12 +864,6 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) {
 				"DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386",
 				"install_dev",
 			},
-		}, {
-			Env: []string{},
-			Argv: []string{
-				"rm", "-rf",
-				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386/lib/pkgconfig",
-			},
 		}, {
 			Env: []string{},
 			Argv: []string{
@@ -937,12 +919,6 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) {
 				"DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64",
 				"install_dev",
 			},
-		}, {
-			Env: []string{},
-			Argv: []string{
-				"rm", "-rf",
-				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64/lib/pkgconfig",
-			},
 		}},
 	}}
 
@@ -1045,6 +1021,7 @@ func TestAndroidBuildCdepsLibevent(t *testing.T) {
 					"CXXFLAGS=-fdata-sections -ffunction-sections -fstack-protector-strong -funwind-tables -no-canonical-prefixes -D_FORTIFY_SOURCE=2 -fpic -mthumb -Oz -DANDROID",
 					"-I"+faketopdir+"/internal/cmd/buildtool/internal/libtor/android/arm/include",
 				),
+				"PKG_CONFIG_PATH=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm/lib/pkgconfig",
 			},
 			Argv: []string{
 				"./configure",
@@ -1077,8 +1054,36 @@ func TestAndroidBuildCdepsLibevent(t *testing.T) {
 			Env: []string{},
 			Argv: []string{
 				"rm",
-				"-rf",
-				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm/lib/pkgconfig",
+				"-f",
+				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm/lib/pkgconfig/libevent.pc",
+			},
+		}, {
+			Env: []string{},
+			Argv: []string{
+				"rm",
+				"-f",
+				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm/lib/pkgconfig/libevent_core.pc",
+			},
+		}, {
+			Env: []string{},
+			Argv: []string{
+				"rm",
+				"-f",
+				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm/lib/pkgconfig/libevent_extra.pc",
+			},
+		}, {
+			Env: []string{},
+			Argv: []string{
+				"rm",
+				"-f",
+				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm/lib/pkgconfig/libevent_openssl.pc",
+			},
+		}, {
+			Env: []string{},
+			Argv: []string{
+				"rm",
+				"-f",
+				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm/lib/pkgconfig/libevent_pthreads.pc",
 			},
 		}, {
 			Env: []string{},
@@ -1195,6 +1200,7 @@ func TestAndroidBuildCdepsLibevent(t *testing.T) {
 					"CXXFLAGS=-fdata-sections -ffunction-sections -fstack-protector-strong -funwind-tables -no-canonical-prefixes -D_FORTIFY_SOURCE=2 -fpic -O2 -DANDROID",
 					"-I"+faketopdir+"/internal/cmd/buildtool/internal/libtor/android/arm64/include",
 				),
+				"PKG_CONFIG_PATH=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64/lib/pkgconfig",
 			},
 			Argv: []string{
 				"./configure",
@@ -1227,8 +1233,36 @@ func TestAndroidBuildCdepsLibevent(t *testing.T) {
 			Env: []string{},
 			Argv: []string{
 				"rm",
-				"-rf",
-				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64/lib/pkgconfig",
+				"-f",
+				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64/lib/pkgconfig/libevent.pc",
+			},
+		}, {
+			Env: []string{},
+			Argv: []string{
+				"rm",
+				"-f",
+				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64/lib/pkgconfig/libevent_core.pc",
+			},
+		}, {
+			Env: []string{},
+			Argv: []string{
+				"rm",
+				"-f",
+				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64/lib/pkgconfig/libevent_extra.pc",
+			},
+		}, {
+			Env: []string{},
+			Argv: []string{
+				"rm",
+				"-f",
+				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64/lib/pkgconfig/libevent_openssl.pc",
+			},
+		}, {
+			Env: []string{},
+			Argv: []string{
+				"rm",
+				"-f",
+				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64/lib/pkgconfig/libevent_pthreads.pc",
 			},
 		}, {
 			Env: []string{},
@@ -1345,6 +1379,7 @@ func TestAndroidBuildCdepsLibevent(t *testing.T) {
 					"CXXFLAGS=-fdata-sections -ffunction-sections -fstack-protector-strong -funwind-tables -no-canonical-prefixes -D_FORTIFY_SOURCE=2 -fPIC -O2 -DANDROID -mstackrealign",
 					"-I"+faketopdir+"/internal/cmd/buildtool/internal/libtor/android/386/include",
 				),
+				"PKG_CONFIG_PATH=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386/lib/pkgconfig",
 			},
 			Argv: []string{
 				"./configure",
@@ -1377,8 +1412,36 @@ func TestAndroidBuildCdepsLibevent(t *testing.T) {
 			Env: []string{},
 			Argv: []string{
 				"rm",
-				"-rf",
-				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386/lib/pkgconfig",
+				"-f",
+				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386/lib/pkgconfig/libevent.pc",
+			},
+		}, {
+			Env: []string{},
+			Argv: []string{
+				"rm",
+				"-f",
+				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386/lib/pkgconfig/libevent_core.pc",
+			},
+		}, {
+			Env: []string{},
+			Argv: []string{
+				"rm",
+				"-f",
+				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386/lib/pkgconfig/libevent_extra.pc",
+			},
+		}, {
+			Env: []string{},
+			Argv: []string{
+				"rm",
+				"-f",
+				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386/lib/pkgconfig/libevent_openssl.pc",
+			},
+		}, {
+			Env: []string{},
+			Argv: []string{
+				"rm",
+				"-f",
+				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386/lib/pkgconfig/libevent_pthreads.pc",
 			},
 		}, {
 			Env: []string{},
@@ -1495,6 +1558,7 @@ func TestAndroidBuildCdepsLibevent(t *testing.T) {
 					"CXXFLAGS=-fdata-sections -ffunction-sections -fstack-protector-strong -funwind-tables -no-canonical-prefixes -D_FORTIFY_SOURCE=2 -fPIC -O2 -DANDROID",
 					"-I"+faketopdir+"/internal/cmd/buildtool/internal/libtor/android/amd64/include",
 				),
+				"PKG_CONFIG_PATH=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64/lib/pkgconfig",
 			},
 			Argv: []string{
 				"./configure",
@@ -1527,8 +1591,36 @@ func TestAndroidBuildCdepsLibevent(t *testing.T) {
 			Env: []string{},
 			Argv: []string{
 				"rm",
-				"-rf",
-				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64/lib/pkgconfig",
+				"-f",
+				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64/lib/pkgconfig/libevent.pc",
+			},
+		}, {
+			Env: []string{},
+			Argv: []string{
+				"rm",
+				"-f",
+				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64/lib/pkgconfig/libevent_core.pc",
+			},
+		}, {
+			Env: []string{},
+			Argv: []string{
+				"rm",
+				"-f",
+				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64/lib/pkgconfig/libevent_extra.pc",
+			},
+		}, {
+			Env: []string{},
+			Argv: []string{
+				"rm",
+				"-f",
+				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64/lib/pkgconfig/libevent_openssl.pc",
+			},
+		}, {
+			Env: []string{},
+			Argv: []string{
+				"rm",
+				"-f",
+				faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64/lib/pkgconfig/libevent_pthreads.pc",
 			},
 		}, {
 			Env: []string{},

internal/cmd/buildtool/cdepslibevent.go

@@ -49,6 +49,10 @@ func cdepsLibeventBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependenci
 	}
 	envp := cBuildExportAutotools(cBuildMerge(globalEnv, localEnv))
 
+	// On iOS, we need PKG_CONFIG_PATH to convince libevent to use the OpenSSL we built and
+	// always letting libevent's configure use pkgconfig is actually fine.
+	envp.Append("PKG_CONFIG_PATH", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig"))
+
 	argv := runtimex.Try1(shellx.NewArgv("./configure"))
 	if globalEnv.CONFIGURE_HOST != "" {
 		argv.Append("--host=" + globalEnv.CONFIGURE_HOST)
@@ -59,7 +63,17 @@ func cdepsLibeventBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependenci
 	must.Run(log.Log, "make", "V=1", "-j", strconv.Itoa(runtime.NumCPU()))
 	must.Run(log.Log, "make", "DESTDIR="+globalEnv.DESTDIR, "install")
 	must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.DESTDIR, "bin"))
-	must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig"))
+
+	// We used to delete the whole pkgconfig directory but libevent's build needs OpenSSL's
+	// pkgconfig, so removing the whole directory means rebuilding libevent requires building
+	// OpenSSL because its pkgconfig is also removed. We discovered this need when working
+	// on the https://github.com/ooni/probe-cli/pull/1366 MVP. To keep the build idempotent,
+	// let's be more gentle and just remove libevent's pkgconfig files instead.
+	must.Run(log.Log, "rm", "-f", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig", "libevent.pc"))
+	must.Run(log.Log, "rm", "-f", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig", "libevent_core.pc"))
+	must.Run(log.Log, "rm", "-f", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig", "libevent_extra.pc"))
+	must.Run(log.Log, "rm", "-f", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig", "libevent_openssl.pc"))
+	must.Run(log.Log, "rm", "-f", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig", "libevent_pthreads.pc"))
 
 	// we just need libevent.a
 	must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.DESTDIR, "lib", "libevent.la"))

internal/cmd/buildtool/cdepsopenssl.go

@@ -82,5 +82,8 @@ func cdepsOpenSSLBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependencie
 	))
 
 	must.Run(log.Log, "make", "DESTDIR="+globalEnv.DESTDIR, "install_dev")
-	must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig"))
+
+	// We used to delete the pkgconfig but it turns out this is important for libevent iOS builds, which
+	// means now we need to keep it. See https://github.com/ooni/probe-cli/pull/1369 for details.
+	//must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig"))
 }

internal/cmd/buildtool/linuxcdeps_test.go

@@ -136,12 +136,6 @@ func TestLinuxCdepsBuildMain(t *testing.T) {
 				"DESTDIR=" + faketopdir + "/" + sysDepDestDir,
 				"install_dev",
 			},
-		}, {
-			Env: []string{},
-			Argv: []string{
-				"rm", "-rf",
-				faketopdir + "/" + sysDepDestDir + "/lib/pkgconfig",
-			},
 		}},
 	}, {
 		name:   "we can build libevent",
@@ -195,6 +189,7 @@ func TestLinuxCdepsBuildMain(t *testing.T) {
 					faketopdir,
 					sysDepDestDir,
 				),
+				"PKG_CONFIG_PATH=" + faketopdir + "/" + sysDepDestDir + "/lib/pkgconfig",
 			},
 			Argv: []string{
 				"./configure",
@@ -226,8 +221,36 @@ func TestLinuxCdepsBuildMain(t *testing.T) {
 			Env: []string{},
 			Argv: []string{
 				"rm",
-				"-rf",
-				faketopdir + "/" + sysDepDestDir + "/lib/pkgconfig",
+				"-f",
+				faketopdir + "/" + sysDepDestDir + "/lib/pkgconfig/libevent.pc",
+			},
+		}, {
+			Env: []string{},
+			Argv: []string{
+				"rm",
+				"-f",
+				faketopdir + "/" + sysDepDestDir + "/lib/pkgconfig/libevent_core.pc",
+			},
+		}, {
+			Env: []string{},
+			Argv: []string{
+				"rm",
+				"-f",
+				faketopdir + "/" + sysDepDestDir + "/lib/pkgconfig/libevent_extra.pc",
+			},
+		}, {
+			Env: []string{},
+			Argv: []string{
+				"rm",
+				"-f",
+				faketopdir + "/" + sysDepDestDir + "/lib/pkgconfig/libevent_openssl.pc",
+			},
+		}, {
+			Env: []string{},
+			Argv: []string{
+				"rm",
+				"-f",
+				faketopdir + "/" + sysDepDestDir + "/lib/pkgconfig/libevent_pthreads.pc",
 			},
 		}, {
 			Env: []string{},