From 1c196037d66cdf0c7ace9b37ad48a0913c6156d9 Mon Sep 17 00:00:00 2001 From: Patrice Chalin Date: Mon, 10 Sep 2018 12:10:00 -0400 Subject: [PATCH] Setup use of site-shared, drop local copies of setup files (#1256) * Setup use of site-shared, drop local copies of setup files * Final synlink rename --- .gitmodules | 4 + .travis.yml | 27 +++--- README.md | 2 +- _config.yml | 6 +- site-shared | 1 + src/_assets/images/shared | 1 + src/_plugins/regex_replace_filter.rb | 10 +- src/_plugins/symlinked_sources_listener.rb | 1 + tool/before-install.sh | 1 + tool/{check-examples.sh => check-code.sh} | 0 tool/check-links.sh | 57 ----------- tool/deploy.sh | 42 -------- tool/env-set.sh | 17 +--- tool/install.sh | 31 +----- tool/serve.sh | 71 +------------- tool/shared | 1 + tool/shared/env-set.sh | 108 --------------------- tool/shared/get-ruby.sh | 32 ------ 18 files changed, 33 insertions(+), 379 deletions(-) create mode 100644 .gitmodules create mode 160000 site-shared create mode 120000 src/_assets/images/shared mode change 100644 => 120000 src/_plugins/regex_replace_filter.rb create mode 120000 src/_plugins/symlinked_sources_listener.rb create mode 120000 tool/before-install.sh rename tool/{check-examples.sh => check-code.sh} (100%) delete mode 100755 tool/check-links.sh delete mode 100755 tool/deploy.sh mode change 100755 => 120000 tool/install.sh mode change 100755 => 120000 tool/serve.sh create mode 120000 tool/shared delete mode 100644 tool/shared/env-set.sh delete mode 100644 tool/shared/get-ruby.sh diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..7b4948bf9b --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "site-shared"] + path = site-shared + url = https://github.com/dart-lang/site-shared.git + branch = chalin-zsh-0810 diff --git a/.travis.yml b/.travis.yml index 5754b450f6..e64e651e6b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,13 +11,21 @@ addons: - lib32stdc++6 - libstdc++6 +cache: + timeout: 1000 + bundler: true + directories: + - $HOME/.pub-cache + - $HOME/.nvm + - $HOME/.rvm + - node_modules + env: global: - JEKYLL_ENV=production - TZ=US/Pacific # normalize build timestamp matrix: - - TASK="./tool/check-examples.sh" - # SKIP link checking until we can get linkcheck running + - TASK="./tool/check-code.sh" - TASK="bundle exec jekyll build" before_install: @@ -26,16 +34,16 @@ before_install: install: - ./tool/install.sh - # - ./tool/write-ci-info.sh -v + # - ./tool/shared/write-ci-info.sh -v script: $TASK after_script: - - if [[ $TASK == *build* ]]; then ./tool/check-links.sh || travis_terminate 1; fi + - if [[ $TASK == *build* ]]; then ./tool/shared/check-links.sh || travis_terminate 1; fi deploy: - provider: script - script: ./tool/deploy.sh dash # TODO: drop dash + script: ./tool/shared/deploy.sh dash # TODO: drop dash skip_cleanup: true on: repo: flutter/website @@ -45,12 +53,3 @@ deploy: # Only run Travis jobs for named branches (to avoid double builds for each PR) branches: only: [master, dash, /^\d\.x$/, /^\d-dev$/, /travis-build/] - -cache: - timeout: 1000 - bundler: true - directories: - - $HOME/.pub-cache - - $HOME/.nvm - - $HOME/.rvm - - node_modules diff --git a/README.md b/README.md index 91c9664421..c4e323b541 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ if you already have the required packages installed. - `bundle exec jekyll serve --incremental --watch --livereload --port 4002` 1. Prior to submitting, run link validation:
- `tool/check-links.sh` + `tool/shared/check-links.sh` ## Deploy to a staging site diff --git a/_config.yml b/_config.yml index debb29bf05..a79942fb91 100644 --- a/_config.yml +++ b/_config.yml @@ -20,9 +20,13 @@ defaults: # Build settings source: src +# `symlinked-sources` can refer to individual files or directories +# under `src` that are symlinked to somewhere outside `src`, or it can +# refer to the root folder of the symlinked content (like `site-shared/src`): +symlinked-sources: [site-shared/src] plugins: - - jekyll-redirect-from + - jekyll-redirect-from # TODO(chalin): drop this in favor of Firebase redirects ## Site-wide shorthands diff --git a/site-shared b/site-shared new file mode 160000 index 0000000000..95a80a71dc --- /dev/null +++ b/site-shared @@ -0,0 +1 @@ +Subproject commit 95a80a71dcb6a4c75ece1a04068413f366b03e50 diff --git a/src/_assets/images/shared b/src/_assets/images/shared new file mode 120000 index 0000000000..2da6015cdd --- /dev/null +++ b/src/_assets/images/shared @@ -0,0 +1 @@ +../../../site-shared/src/_assets/images \ No newline at end of file diff --git a/src/_plugins/regex_replace_filter.rb b/src/_plugins/regex_replace_filter.rb deleted file mode 100644 index 97dbb72f93..0000000000 --- a/src/_plugins/regex_replace_filter.rb +++ /dev/null @@ -1,9 +0,0 @@ -# Copied from https://github.com/dart-lang/site-shared/blob/master/src/_plugins/regex_replace_filter.rb -module RegexReplaceFilter - # From https://github.com/Shopify/liquid/issues/202#issuecomment-19112872 - def regex_replace(input, regex, replacement = '') - input.to_s.gsub(Regexp.new(regex), replacement.to_s) - end -end - -Liquid::Template.register_filter(RegexReplaceFilter) diff --git a/src/_plugins/regex_replace_filter.rb b/src/_plugins/regex_replace_filter.rb new file mode 120000 index 0000000000..43fbaa4deb --- /dev/null +++ b/src/_plugins/regex_replace_filter.rb @@ -0,0 +1 @@ +../../site-shared/src/_plugins/regex_replace_filter.rb \ No newline at end of file diff --git a/src/_plugins/symlinked_sources_listener.rb b/src/_plugins/symlinked_sources_listener.rb new file mode 120000 index 0000000000..ac92b66cd6 --- /dev/null +++ b/src/_plugins/symlinked_sources_listener.rb @@ -0,0 +1 @@ +../../site-shared/src/_plugins/symlinked_sources_listener.rb \ No newline at end of file diff --git a/tool/before-install.sh b/tool/before-install.sh new file mode 120000 index 0000000000..7ad90c7c9a --- /dev/null +++ b/tool/before-install.sh @@ -0,0 +1 @@ +shared/before-install.sh \ No newline at end of file diff --git a/tool/check-examples.sh b/tool/check-code.sh similarity index 100% rename from tool/check-examples.sh rename to tool/check-code.sh diff --git a/tool/check-links.sh b/tool/check-links.sh deleted file mode 100755 index eff6d716e3..0000000000 --- a/tool/check-links.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -# -# Tweaked a copy of https://github.com/dart-lang/site-shared/blob/master/tool/serve.sh - -set -e -o pipefail - -[[ -z "$DART_SITE_ENV_DEFS" ]] && . ./tool/env-set.sh - -PORT=${SITE_LOCALHOST_PORT:-4002} - -set +e -SITE=$(grep '^destination:' _config.yml | awk '{ print $2}') -set -e -: ${SITE:-_site} - -if [ ! -e "./$SITE" ]; then - echo "INFO: $SITE directory not found. Site not built? Skipping link checks." - exit 0 -fi - -set -x -npx superstatic --port $PORT 2>&1 | tee > "$TMP/superstatic-log.txt" & -set +x -SERVER_PID=$! -sleep 2 -if [[ -n "$TRAVIS" ]]; then sleep 2; fi - -if ! kill -0 $SERVER_PID > /dev/null 2>&1; then - echo; echo "WARNING: Failed to launch superstatic server. I'll assume it is already running." - if [[ -n "$TRAVIS" ]]; then - echo "Here's the report error (that we're ignoring):"; echo - cat "$TMP/superstatic-log.txt" - fi - echo - SERVER_PID= -else - echo "Server PID: $SERVER_PID" -fi - -# Don't check for external links yet since it seems to cause problems on Travis: --external -set -x -pub run linkcheck \ - --skip-file ./tool/config/linkcheck-skip-list.txt \ - :$PORT \ - | tee "$TMP/linkcheck-log.txt" -set +x - -if ! grep '^\s*0 errors' "$TMP/linkcheck-log.txt" | wc -l > /dev/null; then - CHECK_EXIT_CODE=1 - echo; cat "$TMP/superstatic-log.txt"; echo -else - CHECK_EXIT_CODE=0 -fi - -if [[ -n $SERVER_PID ]]; then kill $SERVER_PID; fi - -exit $CHECK_EXIT_CODE diff --git a/tool/deploy.sh b/tool/deploy.sh deleted file mode 100755 index 7a344d8236..0000000000 --- a/tool/deploy.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash - -if [ ! $(type -t travis_fold) ]; then travis_fold () { true; } fi - -function _error() { echo "ERROR: $1"; exit 1; } - -function _usage() { - echo "Usage: $(basename $0) [--help] [--local] [--quiet] [firebase-project-name]"; - echo - echo " If unspecified, default firebase project is 'default'." - echo - echo " --local Deploy locally from an interactive shell where you are" - echo " already authenticated and logged in with firebase." - echo - exit 0 -} - -while [[ "$1" == -* ]]; do - case "$1" in - -h|--help) _usage;; - -l|--local) LOCAL=1; shift;; - -q|--quiet) QUIET=1; shift;; - *) _error "Unrecognized option: $1. Use --help for details.";; - esac -done - -_ARGS="" -_FB_PROJ=${1:-default} - -if [[ -z $LOCAL && -z "$FIREBASE_TOKEN" ]]; then - _error "Cannot deploy, the FIREBASE_TOKEN environment variable isn't defined." -elif [[ -z $LOCAL ]]; then - _ARGS+=" --non-interactive --token $FIREBASE_TOKEN" -fi - -_ARGS+=" --project $_FB_PROJ" - -if [[ -z $QUIET ]]; then - echo "Deploying to Firebase project: $_FB_PROJ" - set -x # Travis masks out secrets from logs so enabling command echo is safe. -fi -npx firebase deploy $_ARGS diff --git a/tool/env-set.sh b/tool/env-set.sh index 7c38c0c50f..c8ce68f28a 100644 --- a/tool/env-set.sh +++ b/tool/env-set.sh @@ -1,18 +1,5 @@ -# This bash file is meant to be source'd, not executed. +# This bash file is meant to be source'd from the repo root, not executed. source tool/shared/env-set.sh -# Temporary, until we start using site-shared: - -# nvm install 10 -# nvm use 10 -# rvm install 2.4.3 -# rvm use 2.4.3 - -# function travis_fold () { true; } -# export -f travis_fold - -# : ${TMP:=tmp} -# [[ -e "$TMP" ]] || mkdir -pv "$TMP" - -# Site specific settings here: currently, none. +# Site-specific settings here: currently, none. diff --git a/tool/install.sh b/tool/install.sh deleted file mode 100755 index ca966d6782..0000000000 --- a/tool/install.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -set -e -o pipefail - -[[ -z "$DART_SITE_ENV_DEFS" ]] && . ./tool/env-set.sh - -cd `dirname $0`/.. - -if [[ "$(node --version)" < "v10" ]]; then - echo "ERROR: bad version of node detected. If you have nvm installed, type:" - echo " nvm use" - echo "Aborting installation." - exit 1; -else - echo "Node version: $(node --version)" -fi - -if ! __type_t travis_fold; then travis_fold() { true; } fi # For ZSH users - -travis_fold start install.npm_install - (set -x; npm install --no-optional) -travis_fold end install.npm_install - -if [[ -z "$TRAVIS" || "$TASK" == *build* ]]; then - travis_fold start install.bundle - (set -x; bundle install) - travis_fold end install.bundle -fi - -echo "Installation done" diff --git a/tool/install.sh b/tool/install.sh new file mode 120000 index 0000000000..87ffa748a6 --- /dev/null +++ b/tool/install.sh @@ -0,0 +1 @@ +shared/install.sh \ No newline at end of file diff --git a/tool/serve.sh b/tool/serve.sh deleted file mode 100755 index 39b5cbeb63..0000000000 --- a/tool/serve.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash -# -# Tweaked a copy of https://github.com/dart-lang/site-shared/blob/master/tool/serve.sh - -set -e -o pipefail - -# Assume that this script will be linked to ~/scripts/foo or ~/tool/foo in the parent repo. -cd `dirname $0`/.. - -SERVE="npx superstatic" -JEKYLL_OPTS="--incremental --watch " - -while [[ "$1" == -* ]]; do - case "$1" in - --clean) CLEAN=1; shift;; - --dev) FILE=_config_dev.yml - if [[ -e $FILE ]]; then - CONFIG=",$FILE$CONFIG" - else - echo "Warning: $1 option ignored because $FILE not found" - fi - shift;; - --firebase) SERVE="npx firebase serve"; - shift;; - -h|--help) echo "Usage: $(basename $0) [options]"; - echo - echo " --clean Delete the content of site.g before building the site." - echo " --dev Build using _config_dev.yml." - echo " --firebase Serve using firebase rather than superstatic." - echo " --help Print this usage text." - echo " --pin-now Build using _config_now.yml." - echo " --trace Run jekyll build with --trace." - exit 0;; - --pin-now) FILE=_config_now.yml - if [[ -e $FILE ]]; then - CONFIG=",$FILE$CONFIG" - else - echo "Warning: $1 option ignored because $FILE not found" - fi - shift;; - --trace) JEKYLL_OPTS+="--trace "; - shift;; - *) echo "ERROR: Unrecognized option: $1. Use --help for details."; - exit 1;; - esac -done - -if [[ -n $CONFIG ]]; then - CONFIG="--config _config.yml$CONFIG" -fi - -set +e -DEST=$(grep '^destination:' _config.yml | awk '{ print $2}') -set -e -SITE_PATH="./${DEST:-_site}" -if [[ -n $CLEAN ]]; then - if [[ -e "$SITE_PATH" ]]; then - (set -x; rm -Rf "$SITE_PATH/*") - else - echo "WARNING: $SITE_PATH doesn't exist, so there is nothing to clean."; echo - fi -fi - -(set -x; bundle exec jekyll build $CONFIG $JEKYLL_OPTS) & -j_pid=$! -(set -x; $SERVE --version; $SERVE --port ${SITE_LOCALHOST_PORT:-4002}) & -f_pid=$! -echo "Cached PIDs for build and serve: $j_pid, $f_pid" -trap "{ kill $j_pid; kill $f_pid; exit 0;}" SIGINT -wait diff --git a/tool/serve.sh b/tool/serve.sh new file mode 120000 index 0000000000..b34940cbbc --- /dev/null +++ b/tool/serve.sh @@ -0,0 +1 @@ +shared/serve.sh \ No newline at end of file diff --git a/tool/shared b/tool/shared new file mode 120000 index 0000000000..d2071e134d --- /dev/null +++ b/tool/shared @@ -0,0 +1 @@ +../site-shared/tool/ \ No newline at end of file diff --git a/tool/shared/env-set.sh b/tool/shared/env-set.sh deleted file mode 100644 index b2de622c4b..0000000000 --- a/tool/shared/env-set.sh +++ /dev/null @@ -1,108 +0,0 @@ -# Temporary: copy of site-shared/tool/shared/env-set.sh -# -# - This script will be sourced from the host repo's REPO_ROOT/tool folder. -# - REPO_ROOT/scripts/shared symlinks to site-shared/tool. -# - REPO_ROOT contains `_config.yml` with a `port` field. - -_DART_SITE_ENV_SET_INSTALL_OPT="--install" - -if [[ $# -gt 0 ]]; then - case "$1" in - -h|--help) echo "source scripts/env-set.sh [--quick|--no-install]" - echo - echo " Runs nvm and rvm commands 'install' and 'use' to get required Node" - echo " and Ruby versions; then sets other required environment variables." - echo - echo " --no-install Skips nvm and rvm 'install' in favor of 'use'." - echo " --quick Do nothing if this script has already been sourced." - echo - return 0;; - -q|--quick) _DART_SITE_ENV_SET_INSTALL_OPT="--quick"; shift;; - -n|--no-install) - _DART_SITE_ENV_SET_INSTALL_OPT="--no-install"; shift;; - *) echo "ERROR: Unrecognized option for env-set.sh: $1. Use --help for details." - return 1;; - esac -fi - -if [[ $# -gt 0 ]]; then - echo "ERROR: too many arguments passed to env-set.sh: $1. Use --help for details." - return 1 -fi - -if [[ "$_DART_SITE_ENV_SET_INSTALL_OPT" != "--quick" ]]; then - unset DART_SITE_ENV_DEFS -fi - -__type_t() { if [[ -n "$ZSH_VERSION" ]]; then whence -w $*; else type -t $*; fi } -export -f __type_t > /dev/null - -if ! __type_t rvm > /dev/null; then - echo "ERROR: rvm not installed. See README setup instructions. Skipping setup." -elif ! __type_t nvm > /dev/null; then - echo "ERROR: nvm not installed. See README setup instructions. Skipping setup." -elif [[ -z "$DART_SITE_ENV_DEFS" ]]; then - export DART_SITE_ENV_DEFS=1 - export ANSI_YELLOW="\033[33;1m" - export ANSI_RESET="\033[0m" - echo -e "${ANSI_YELLOW}Setting environment variables from tool/env-set.sh${ANSI_RESET}" - - if [[ "$_DART_SITE_ENV_SET_INSTALL_OPT" == "--install" ]]; then - nvm install 10 - else - nvm use 10 - fi - source tool/shared/get-ruby.sh "$_DART_SITE_ENV_SET_INSTALL_OPT" - - if ! __type_t travis_fold > /dev/null; then - # In case this is being run locally. Turn travis_fold into a noop. - travis_fold () { return 0; } - export -f travis_fold > /dev/null - fi - - case "$(uname -a)" in - Darwin\ *) _OS_NAME=macos ;; - Linux\ *) _OS_NAME=linux ;; - *) _OS_NAME=linux ;; - esac - export _OS_NAME - - : ${TMP:=$HOME/tmp} - : ${PKG:=$TMP/pkg} - export TMP - export PKG - - if [[ -n "$TRAVIS" ]]; then - [[ ! -d "$TMP" ]] && mkdir "$TMP" - [[ ! -d "$PKG" ]] && mkdir "$PKG" - else - if [[ -z "$(__type_t dart)" && ! $PATH =~ \/dart-sdk ]]; then - export DART_SDK="$PKG/dart-sdk" - # Updating PATH to include access to Dart bin. - export PATH="$PATH:$DART_SDK/bin" - export PATH="$PATH:$HOME/.pub-cache/bin" - fi - fi - - export SITE_LOCALHOST_PORT=5000 - if [[ ! -e _config.yml ]]; then - echo "WARNING: '_config.yml' file not found; setting SITE_LOCALHOST_PORT to $SITE_LOCALHOST_PORT." - else - export SITE_LOCALHOST_PORT=$(grep '^port:' _config.yml | awk '{ print $2}') - if [[ -z $SITE_LOCALHOST_PORT ]]; then - echo "WARNING: '_config.yml' file has no 'port' field; setting SITE_LOCALHOST_PORT to $SITE_LOCALHOST_PORT."; - fi - fi - - if [[ -z $(git config push.recurseSubmodules) ]]; then - echo "INFO: git config push.recurseSubmodules is unset. Setting it to 'check':" - (set -x; git config push.recurseSubmodules check) - else - echo "INFO: git config push.recurseSubmodules is set to $(git config push.recurseSubmodules)." - fi - git config status.submodulesummary 1 - echo "INFO: git config status.submodulesummary is set to $(git config status.submodulesummary)." - -fi - -return 0 diff --git a/tool/shared/get-ruby.sh b/tool/shared/get-ruby.sh deleted file mode 100644 index faa83d4afa..0000000000 --- a/tool/shared/get-ruby.sh +++ /dev/null @@ -1,32 +0,0 @@ -# This bash file is meant to be source'd, not executed. It is a bit non-DRY -# because we can't define local variables, nor can we use functions. - -DART_SITE_RUBY_VERS=ruby-2.4.3 - -if ! rvm version | grep -q '(latest)'; then - rvm get stable -fi - -echo "RVM current: $(rvm current)" -if rvm current | grep -Fq $DART_SITE_RUBY_VERS; then - echo " Using $(rvm current) (rvm $(rvm version | awk '{print $2}'))" -elif rvm list | grep -Fq $DART_SITE_RUBY_VERS; then - if [[ -n "$TRAVIS" ]]; then DEFAULT="--default "; fi - echo "Running: rvm ${DEFAULT}use $DART_SITE_RUBY_VERS" - rvm ${DEFAULT}use $DART_SITE_RUBY_VERS -elif [[ -z "$1" || "$1" == "--install" ]]; then - echo "Running: rvm install $DART_SITE_RUBY_VERS" - rvm install $DART_SITE_RUBY_VERS -else - echo "ERROR: ruby '$DART_SITE_RUBY_VERS' isn't locally available and --install option was not specified." - return 1 -fi - -echo "Ruby --version: $(ruby --version)" - -if [[ ! -e ~/.rvmrc ]] || ! grep -q rvm_silence_path_mismatch_check_flag ~/.rvmrc; then - echo rvm_silence_path_mismatch_check_flag=1 >> ~/.rvmrc - echo "Configured RVM so it doesn't complain if it isn't first in PATH" -fi - -return 0