diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..8494b290 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.vagrant +tmp/ +debian/*.debhelper.log +debian/cedar/ +debian/files +debian/*.substvars diff --git a/Readme.md b/Readme.md new file mode 100644 index 00000000..33dbc9b5 --- /dev/null +++ b/Readme.md @@ -0,0 +1,45 @@ +Heroku Stack Images +========= + +The provided bin/cedar.sh is the basis of a cedar stack image. + + cd hvm + vagrant up + vagrant ssh + + sudo /vagrant/bin/build-stack 2.0.0 /vagrant/bin/cedar.sh + -----> Starting build + -----> Installing build tools + Hit http://us.archive.ubuntu.com lucid Release.gpg + ... + -----> Cleaning up. Logs in /tmp/log/build-stack.log + + sudo /vagrant/bin/capture-stack 2.0.0 + -----> Starting capture + -----> Creating image file /tmp/cedar64-2.0.0.img + 24+0 records in + ... + -----> Cleaning up. Logs in /tmp/log/capture-stack.log + + sudo /vagrant/bin/install-stack /tmp/cedar64-2.0.0.img.gz + -----> Starting install + -----> Gunzipping image /tmp/cedar64-2.0.0.img.gz + -----> Mounting image /mnt/stacks/cedar64-2.0.0 + +Stack images are generally pushed to S3 and installed from there. + + sudo /vagrant/bin/push-stack 2.0.0 stacks_bucket \ + AWS_ACCESS_KEY_ID=xxx AWS_SECRET_ACCESS_KEY=xxx + -----> Starting push + -----> Uploading files + /tmp/cedar64-2.0.0.img.gz -> s3://stacks_bucket/cedar64-2.0.0.img.gz + ... + -----> Signing URLs + http://s3.amazonaws.com/noah_herokustacks/cedar64-2.0.0.img.gz?AWS... + http://s3.amazonaws.com/noah_herokustacks/cedar64-2.0.0.img.md5 + + sudo /vagrant/bin/install-stack \ + "http://s3.amazonaws.com/noah_herokustacks/cedar64-2.0.0.img.gz?AWS..." + -----> Starting install + -----> Downloading and gunzipping image + -----> Mounting image /mnt/stacks/cedar64-2.0.0 diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 00000000..0b30eb35 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,19 @@ +Vagrant::Config.run do |config| + config.vm.box = "lucid64" + config.vm.box_url = "http://files.vagrantup.com/lucid64.box" + config.vm.network :hostonly, "33.33.33.3" + config.ssh.forward_agent = true + config.vm.share_folder("log", "/tmp/log", "tmp/log") + + config.vm.provision :shell, :inline => <<-'EOSRC' +( + date + + apt-get update + apt-get install -y curl git-core lxc s3cmd + mkdir -p /cgroup + mount none -t cgroup /cgroup + +) 2>&1 | tee /tmp/log/provision.log +EOSRC +end \ No newline at end of file diff --git a/bin/build-stack b/bin/build-stack new file mode 100755 index 00000000..aa96eb04 --- /dev/null +++ b/bin/build-stack @@ -0,0 +1,45 @@ +#!/bin/bash +source $(dirname $0)/stack-helpers.sh +LOG=/tmp/log/$(basename $0).log +mkdir -p /tmp/log + +( + [ $# -eq 2 ] || abort usage: $(basename $0) VERSION SCRIPT + + VERSION=$1 + SCRIPT=$2 + MNT=/tmp/cedar64-$VERSION-build + + [ $UID = 0 ] || abort fatal: must be called with sudo + [[ -f $SCRIPT ]] || abort fatal: script "$SCRIPT" not found + [[ -d $MNT ]] && abort fatal: dir "$MNT" already exists + + function on_exit() { + display Cleaning up. Logs at $LOG + umount $MNT/dev $MNT/proc 2>&1 | indent + } + trap on_exit EXIT + + display Starting build at $(date) + display Installing build tools + ( + apt-get update + apt-get install -y build-essential debootstrap git-core + ) 2>&1 | indent + + display Bootstrapping build env $MNT + ( + mkdir -p $MNT + debootstrap --variant=minbase lucid $MNT + ) | indent + + display Mounting host in build env + ( + mount --bind /dev $MNT/dev + mount --bind /proc $MNT/proc + cp $SCRIPT $MNT/tmp/build.sh + ) | indent + + display Building inside build env + chroot $MNT /tmp/build.sh | indent +) 2>&1 | tee $LOG diff --git a/bin/capture-stack b/bin/capture-stack new file mode 100755 index 00000000..54fda272 --- /dev/null +++ b/bin/capture-stack @@ -0,0 +1,81 @@ +#!/bin/bash +source $(dirname $0)/stack-helpers.sh +LOG=/tmp/log/$(basename $0).log + +( + [ $# -ge 1 ] || abort usage: $(basename $0) VERSION [--no-gzip] + + VERSION=$1 + NOGZIP=$2 + MNT=/tmp/cedar64-$VERSION-build + IMG=/tmp/cedar64-$VERSION.img + IMG_MNT=/tmp/cedar64-$VERSION + + [ $UID = 0 ] || abort fatal: must be called with sudo + [[ -d $MNT ]] || abort fatal: dir "$MNT" not found + + function on_exit() { + display Cleaning up. Logs at $LOG + ( umount $IMG_MNT 2>&1 || true ) | indent + } + trap on_exit EXIT + + display Starting capture at $(date) + display Creating image file $IMG + ( + mkdir -p $(dirname $IMG) + dd if=/dev/zero of=$IMG bs=100M count=24 + yes | mkfs -t ext3 -m 1 $IMG + tune2fs -c 9999 $IMG + tune2fs -i 9999w $IMG + ) 2>&1 | indent + + display Mounting image $IMG_MNT + mkdir -p $IMG_MNT + mount -o loop,noatime,nodiratime $IMG $IMG_MNT + + display Copying stack to image + ( + cd $MNT + rsync -a --exclude=/lib/modules bin etc lib lib64 sbin usr $IMG_MNT + rsync -a var/lib $IMG_MNT/var + ) | indent + + display Modifying image directories and files + ( + for d in app tmp proc dev var var/log var/tmp home/group_home mnt/slug-compiler; do + mkdir -p $IMG_MNT/$d + done + chmod 755 $IMG_MNT/home/group_home + + echo "127.0.0.1 localhost localhost.localdomain" > $IMG_MNT/etc/hosts + + echo "heroku-runtime" > $IMG_MNT/etc/hostname + + for f in etc/profile etc/bash.bashrc; do + echo "export PS1='\\[\\033[01;34m\\]\\w\\[\\033[00m\\] \\[\\033[01;32m\\]$ \\[\\033[00m\\]'" > $IMG_MNT/$f + done + + cat >$IMG_MNT/etc/resolv.conf < $IMG_MNT/home/group_home/.gitconfig <&1 | tee $LOG diff --git a/bin/cedar.sh b/bin/cedar.sh new file mode 100755 index 00000000..25580022 --- /dev/null +++ b/bin/cedar.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +exec 2>&1 +set -e +set -x + +cat > /etc/apt/sources.list < $MNT.img + else + display Downloading and gunzipping image + curl $IMG_GZ | gunzip > $MNT.img + fi + + display Mounting image $MNT + mkdir -p $MNT + mount -o loop,noatime,nodiratime,ro $MNT.img $MNT +) 2>&1 | tee $LOG \ No newline at end of file diff --git a/bin/provision.sh b/bin/provision.sh new file mode 100755 index 00000000..b64a7ea5 --- /dev/null +++ b/bin/provision.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +function append_once() { + while read data; do + grep -q "$data" $1 || echo "$data" >> $1 + done +} + +apt-get update +apt-get -y --force-yes install curl git-core lxc + +# mount cgroup for LXC +mkdir -p /cgroup +mount none -t cgroup /cgroup +append_once /etc/fstab <$S3CFG <&1 | indent | grep ERROR + ) && abort fatal: can not access S3 + + s3cmd -c $S3CFG --progress --acl-private put $IMG_GZ s3://$S3_IMG_GZ + s3cmd -c $S3CFG --progress --acl-public put $IMG_MD5 s3://$S3_IMG_MD5 + + display Signing URLs + ( + # finally we find something bash is miserable at! + EXPIRES=$(date +%s --date "now 100000000 seconds") + SIGNATURE=$(s3cmd -c $S3CFG sign $'GET\n\n\n'$EXPIRES$'\n/'$S3_IMG_GZ) + SIGNATURE=${SIGNATURE#Signature: } + SIGNATURE="$(perl -MURI::Escape -e 'print uri_escape($ARGV[0]);' "$SIGNATURE")" + echo "http://s3.amazonaws.com/$S3_IMG_GZ?AWSAccessKeyId=$AWS_ACCESS_KEY_ID&Expires=$EXPIRES&Signature=$SIGNATURE" + echo "http://s3.amazonaws.com/$S3_IMG_MD5" + ) | indent +) 2>&1 | tee $LOG \ No newline at end of file diff --git a/bin/stack-helpers.sh b/bin/stack-helpers.sh new file mode 100644 index 00000000..237ce2ef --- /dev/null +++ b/bin/stack-helpers.sh @@ -0,0 +1,13 @@ +set -o pipefail + +function display() { + echo -e "\n----->" $* +} + +function abort() { + echo $* ; exit 1 +} + +function indent() { + sed -u "s/^/ /" +} \ No newline at end of file diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 00000000..a7d156eb --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +cedar (1.5.2lucid1) lucid; urgency=low + + * Initial release + + -- Fabio Kung Fri, 12 Oct 2012 10:34:11 -0700 diff --git a/debian/compat b/debian/compat new file mode 100644 index 00000000..7f8f011e --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +7 diff --git a/debian/control b/debian/control new file mode 100644 index 00000000..661c2134 --- /dev/null +++ b/debian/control @@ -0,0 +1,14 @@ +Source: cedar +Section: devel +Priority: optional +Maintainer: Heroku Runtime +Uploaders: Fabio Kung , Noah Zoschke , Phil Hagelberg +Build-Depends: debhelper (>= 7.0.0), build-essential, make (>= 3.81), curl (>= 7.0.0), sudo +Standards-Version: 3.9.3 +Homepage: https://devcenter.heroku.com/articles/stack + +Package: cedar +Architecture: amd64 +Depends: ${misc:Depends}, dpkg (>= 1.16) +Description: Heroku Cedar 64 bits Stack Image. + Base image that powers dyno execution environments (64 bits - amd64). diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 00000000..26223372 --- /dev/null +++ b/debian/copyright @@ -0,0 +1,8 @@ +Format: http://dep.debian.net/deps/dep5 +Upstream-Name: cedar +Source: https://devcenter.heroku.com/articles/stack + +Files: * +Copyright: 2012 Heroku +License: Proprietary + diff --git a/debian/docs b/debian/docs new file mode 100644 index 00000000..e69de29b diff --git a/debian/postinst b/debian/postinst new file mode 100644 index 00000000..f30771fb --- /dev/null +++ b/debian/postinst @@ -0,0 +1,43 @@ +#!/bin/sh +# postinst script for #PACKAGE# +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * `configure' +# * `abort-upgrade' +# * `abort-remove' `in-favour' +# +# * `abort-remove' +# * `abort-deconfigure' `in-favour' +# `removing' +# +# for details, see http://www.debian.org/doc/debian-policy/ or +# the debian-policy package + + +case "$1" in + configure) + sed -i '\_/usr/share/stacks/cedar64-current.img_d' /etc/fstab + mkdir -p /usr/share/stacks/cedar64-current + echo "/usr/share/stacks/cedar64-current.img /usr/share/stacks/cedar64-current ext3 user,noauto,ro,exec,loop 0 0" >> /etc/fstab + mount /usr/share/stacks/cedar64-current || true + ;; + + abort-upgrade|abort-remove|abort-deconfigure) + ;; + + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/debian/prerm b/debian/prerm new file mode 100644 index 00000000..737a7ae5 --- /dev/null +++ b/debian/prerm @@ -0,0 +1,41 @@ +#!/bin/sh +# prerm script for #PACKAGE# +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * `remove' +# * `upgrade' +# * `failed-upgrade' +# * `remove' `in-favour' +# * `deconfigure' `in-favour' +# `removing' +# +# for details, see http://www.debian.org/doc/debian-policy/ or +# the debian-policy package + + +case "$1" in + remove|upgrade|deconfigure) + umount /usr/share/stacks/cedar64-current || true + sed -i '\_/usr/share/stacks/cedar64-current.img_d' /etc/fstab + rmdir /usr/share/stacks/cedar64-current + ;; + + failed-upgrade) + ;; + + *) + echo "prerm called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/debian/rules b/debian/rules new file mode 100755 index 00000000..479f1711 --- /dev/null +++ b/debian/rules @@ -0,0 +1,26 @@ +#!/usr/bin/make -f +# -*- makefile -*- + +# Uncomment this to turn on verbose mode. +#export DH_VERBOSE=1 + +PKGNAME=cedar +PKGDIR=debian/$(PKGNAME) +INSTALLDIR=$(PKGDIR)/usr/share/stacks + +%: + dh $@ + +dist-clean: clean + rm -rf /tmp/$(PKGNAME)* + +binary-arch: + dh_prep + dh_installdirs + test -d /tmp/cedar64-current-build || sudo bin/build-stack current bin/cedar.sh + sudo bin/capture-stack current --no-gzip + sudo chmod a+r /tmp/$(PKGNAME)*.img* + mkdir -p $(INSTALLDIR) + cp /tmp/$(PKGNAME)*.img* $(INSTALLDIR)/ + +binary: binary-arch diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 00000000..89ae9db8 --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (native) diff --git a/debian/source/options b/debian/source/options new file mode 100644 index 00000000..1708ab40 --- /dev/null +++ b/debian/source/options @@ -0,0 +1,4 @@ +tar-ignore = .hg +tar-ignore = .git +tar-ignore = .git* +tar-ignore = .build