Build go-webrtc and snowflake for mac.

tor-browser-bundle.git author: David Fifield <[email protected]> tor-browser-bundle.git commit: 26e0cd44f2886bfad1c3d30844ff7a21eb9d0478 Commit message from the tor-browser-bundle.git commit: Build go-webrtc and snowflake in the mac pluggable-transports descriptor. I had to apply two tricks to get a reproducible snowflake-client. The first is to use faketime to eliminate some timestamps. There were 11 variable timestamps in the file. Through experimentation, I found that 10 of them were dependent on the Go runtime (recompiling Go caused them to change) and 1 was dependent on snowflake-client itself (recompiling snowflake-client with the same runtime changed only that 1 timestamp). The underlying issue has to do with clang 3.8.0 on Darwin embedding timestamps, unsolved in the Go issue tracker as of 13 days ago. golang/go#9206 (comment) The second is a sed command to clobber embedded paths of the form /tmp/go-buildXXXXXXXXX and /tmp/go-link-XXXXXXXXX. Their presence is caused by some combination of Clang and Darwin, and there is as yet no known workaround upstream.
boklm · Jul 19, 2017 · 9b59107 · 9b59107
1 parent de79b7e
commit 9b59107
Show file tree

Hide file tree

Showing 8 changed files with 75 additions and 11 deletions.
diff --git a/projects/go-webrtc/config b/projects/go-webrtc/config
@@ -20,13 +20,23 @@ var:
     # Replace the prebuilt webrtc library with our own one.
     rm -rf include/ lib/
     cp -a /var/tmp/dist/webrtc/{include,lib} .
-    # The go-webrtc package sets _GLIBCXX_USE_CXX11_ABI=0 for compatibility with
-    # the prebuilt libwebrtc-magic.a libraries. Since we build our own
-    # libwebrtc-magic.a with the C++11 ABI, we have to undo that setting here,
-    # using CGO_CXXFLAGS.
-    # __STDC_FORMAT_MACROS is needed for a definition of PRIxPTR from inttypes.h.
-    export CGO_CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=1 -D__STDC_FORMAT_MACROS=1"
-    export CGO_LDFLAGS=-latomic
+    [% IF c("var/linux") -%]
+      # The go-webrtc package sets _GLIBCXX_USE_CXX11_ABI=0 for compatibility with
+      # the prebuilt libwebrtc-magic.a libraries. Since we build our own
+      # libwebrtc-magic.a with the C++11 ABI, we have to undo that setting here,
+      # using CGO_CXXFLAGS.
+      # __STDC_FORMAT_MACROS is needed for a definition of PRIxPTR from inttypes.h.
+      export CGO_CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=1 -D__STDC_FORMAT_MACROS=1"
+      export CGO_LDFLAGS=-latomic
+    [% END -%]
+    [% IF c("var/osx") -%]
+      export CGO_ENABLED=1
+      export CGO_CFLAGS="[% c("var/FLAGS") %] -mmacosx-version-min=10.7"
+      export CGO_CXXFLAGS="[% c("var/FLAGS") %] -stdlib=libc++ -mmacosx-version-min=10.7"
+      export CGO_LDFLAGS="[% c("var/FLAGS") %] -stdlib=libc++ -mmacosx-version-min=10.7"
+      export CC="$clangdir/bin/clang"
+      export CXX="$clangdir/bin/clang++"
+    [% END -%]
 
 targets:
   master:
@@ -36,6 +46,10 @@ targets:
       arch_deps:
         - pkg-config
         - libx11-dev
+  osx-x86_64:
+    var:
+      arch_deps:
+        - pkg-config
 
 input_files:
   - project: container-image

diff --git a/projects/go/build b/projects/go/build
@@ -52,7 +52,12 @@ cd /var/tmp/dist/go/src
   # http://golang.org/doc/install/source#environment
   export GOOS=darwin
   export GOARCH=amd64
-  CGO_ENABLED=1 CC_FOR_TARGET="$CC_FOR_TARGET" CC= CFLAGS= LDFLAGS= ./make.bash
+  # faketime is needed because clang 3.8.0 on Darwin embeds the timestamps of
+  # certain intermediate object files (including those that arise here while
+  # compiling the Go runtime itself). Without this, variable timestamps would
+  # end up in snowflake-client.
+  # https://github.com/golang/go/issues/9206#issuecomment-310476743
+  CGO_ENABLED=1 CC_FOR_TARGET="$CC_FOR_TARGET" CC= CFLAGS= LDFLAGS= [% c("var/faketime") %] ./make.bash
 [% ELSIF c("var/windows") -%]
   # Create a cc-for-target script that closes over CC, CFLAGS, and LDFLAGS.
   # Go's CC_FOR_TARGET only allows a command name, not a command with arguments.

diff --git a/projects/go/config b/projects/go/config
@@ -60,6 +60,8 @@ targets:
     var:
       GOOS: darwin
       GOARCH: amd64
+      arch_deps:
+        - faketime
   linux:
     var:
       GOOS: linux

diff --git a/projects/macosx-toolchain/build b/projects/macosx-toolchain/build
@@ -10,6 +10,8 @@ tar -C $distdir -xf [% c('input_files_by_name/cctools') %]
 cd $distdir/cctools/bin
 ln -s ../../clang/bin/clang x86_64-apple-darwin10-clang
 ln -s ../../clang/bin/clang++ x86_64-apple-darwin10-clang++
+# "go link" expects to find a program called "dsymutil" exactly.
+ln -sf x86_64-apple-darwin10-dsymutil dsymutil
 
 cd "/var/tmp/dist"
 [% c('tar', {

diff --git a/projects/macosx-toolchain/config b/projects/macosx-toolchain/config
@@ -6,7 +6,7 @@ var:
     use_container: 1
   setup: |
     mkdir -p /var/tmp/dist
-    tar -C /var/tmp/dist -xf [% c("compiler_tarfile") %]
+    tar -C /var/tmp/dist -xf $rootdir/[% c("compiler_tarfile") %]
     export sysrootdir=/var/tmp/dist/[% project %]/SDK/
     export clangdir=/var/tmp/dist/[% project %]/clang
     export cctoolsdir=/var/tmp/dist/[% project %]/cctools/bin

diff --git a/projects/snowflake/build b/projects/snowflake/build
@@ -10,6 +10,12 @@ distdir=/var/tmp/dist/[% project %]
 [% IF c("var/osx") %]
   PTDIR="$distdir/Contents/MacOS/Tor/PluggableTransports"
   DOCSDIR="$distdir/Contents/Resources/TorBrowser/Docs/snowflake"
+  export CGO_ENABLED=1
+  export CGO_CFLAGS="[% c("var/FLAGS") %] -mmacosx-version-min=10.7"
+  export CGO_CXXFLAGS="[% c("var/FLAGS") %] -stdlib=libc++ -mmacosx-version-min=10.7"
+  export CGO_LDFLAGS="[% c("var/FLAGS") %] -stdlib=libc++ -mmacosx-version-min=10.7"
+  export CC="$clangdir/bin/clang"
+  export CXX="$clangdir/bin/clang++"
 [% ELSE %]
   PTDIR="$distdir/TorBrowser/Tor/PluggableTransports"
   DOCSDIR="$distdir/TorBrowser/Docs/snowflake"
@@ -25,7 +31,35 @@ tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz
 cd /var/tmp/build/[% project %]-[% c('version') %]
 
 cd client
-go build -ldflags '-s'
+[% IF c("var/osx") -%]
+  # Without faketime, snowflake-client would contain the timestamp of the
+  # temporary client.a file created during "go build".
+  [% c("var/faketime") %] go build -ldflags '-s'
+[% ELSE -%]
+  go build -ldflags '-s'
+[% END -%]
+[% IF c("var/osx") -%]
+  # Hack: Overwrite variable absolute paths embedded in the binary. clang 3.8.0
+  # on Darwin embeds such paths and the issue is unsolved in upstream Go as of
+  # 2016-06-28:
+  # https://github.com/golang/go/issues/9206#issuecomment-310476743
+  # The two kinds of paths are ("000000000" stands for 9 random digits):
+  #   /tmp/go-build000000000
+  #   /tmp/go-link-000000000
+  # Such paths are the output of ioutil.TempDir("", "go-build") and
+  # ioutil.TempDir("", "go-link-").
+  cp -a client client.stomped
+  sed -i -E -e 's#(/tmp/go-build|/tmp/go-link-)[0-9]{9}/#\1XXXXXXXXX/#g' client.stomped
+  # Sanity check: make sure the file actually changed. If it did not, it could
+  # mean that a change in go or clang has made this step unnecessary.
+  if cmp client client.stomped
+  then
+    echo "No paths replaced in snowflake-client. Maybe the replacement failed or is no longer needed."
+    echo "Check projects/snowflake/build"
+    exit 1
+  fi
+  mv -f client.stomped client
+[% END -%]
 cp -a client[% IF c("var/windows") %].exe[% END %] $PTDIR/snowflake-client[% IF c("var/windows") %].exe[% END %]
 
 cd ..

diff --git a/projects/snowflake/config b/projects/snowflake/config
@@ -14,6 +14,10 @@ targets:
       arch_deps:
         - pkg-config
         - libx11-dev
+  osx-x86_64:
+    var:
+      arch_deps:
+        - faketime
 
 input_files:
   - project: container-image

diff --git a/rbm.conf b/rbm.conf
@@ -35,6 +35,7 @@ var:
     [% c("input_files_by_name/" _ file) %]
     [% END -%]
 
+  faketime: "faketime -f \"[% USE date; GET date.format(c('timestamp'), format = '%Y-%m-%d %H:%M:%S') %]\""
   touch: "[% USE date %]touch -m -t [% date.format(c('timestamp'), format = '%Y%m%d%H%M') %]"
 
   locale_ja: ja
@@ -130,7 +131,7 @@ targets:
     var:
       linux: 1
       compiler: gcc
-      # We only build snowflake on linux for now
+      # We only build snowflake on linux and osx for now
       snowflake: 1
       container:
         suite: wheezy
@@ -190,6 +191,8 @@ targets:
       FLAGS: "-target x86_64-apple-darwin10 -mlinker-version=136 -B $cctoolsdir -isysroot $sysrootdir"
       LDFLAGS: "-Wl,-syslibroot,$sysrootdir -Wl,-dead_strip -Wl,-pie"
       locale_ja: ja-JP-mac
+      # We only build snowflake on linux and osx for now
+      snowflake: 1
       deps:
         - build-essential
         - python