X230 flash universal init

.circleci/config.yml

@@ -2,97 +2,186 @@ version: 2
 jobs:
   build:
     docker:
-      - image: osresearch/heads-ubuntu:16.04
+      - image: debian:10
     steps:
       - run:
           name: Install dependencies
           command: |
             apt update
-            apt install -y \
-              build-essential \
-              zlib1g-dev \
-              uuid-dev \
-              libdigest-sha-perl \
-              libelf-dev \
-              bc \
-              bzip2 \
-              bison \
-              flex \
-              git \
-              gnupg \
-              iasl \
-              m4 \
-              nasm \
-              patch \
-              python \
-              wget \
-              gnat \
-              cpio \
-              ccache \
-              lzma \
-
+            apt install -y build-essential zlib1g-dev uuid-dev libdigest-sha-perl libelf-dev bc bzip2 bison flex git gnupg iasl m4 nasm patch python wget gnat cpio ccache pkg-config cmake libusb-1.0-0-dev autoconf texinfo ncurses-dev
       - checkout
 
       - run:
-          name: Bootstrap make
+          name: git reset
           command: |
-            make -j4 bootstrap
+            git reset --hard "$CIRCLE_SHA1" \
 
       - run:
-          name: Bootstrap coreboot-gcc
+          name: Creating all modules and patches digest
           command: |
-            ./build/make-4.2.1/make \
-                TOOLCHAIN=/home/builder/heads \
-                V=1 \
-                BOARD=qemu-coreboot \
-                coreboot-gcc
-
+            find ./patches/ ./modules/ -type f | sort -h |xargs sha256sum > /tmp/all_modules_and_patches.sha256sums \
 
       - run:
-          name: qemu-coreboot
+          name: Creating musl-cross-make and musl-cross-make patches digest
           command: |
-            ./build/make-4.2.1/make \
-                TOOLCHAIN=/home/builder/heads \
-                V=1 \
-                -j4 \
-                BOARD=qemu-coreboot \
+            find ./patches/musl-cross-* modules/musl-cross* -type f | sort -h | xargs sha256sum > /tmp/musl-cross_module_and_patches.sha256sums \
 
-      - store-artifacts:
-          path: build/qemu-coreboot/coreboot.rom
-      - store-artifacts:
-          path: build/qemu-coreboot/hashes.txt
 
+      - restore_cache:
+          keys:
+            #Restore existing cache for modules checksums validated to be exactly the same as in github current commit
+            - heads-modules-and-patches-{{ checksum "/tmp/all_modules_and_patches.sha256sums" }}{{ .Environment.CACHE_VERSION }}
+            #If precedent fails. Restore cache for musl-cross module checksum validated to be exactly the same as in github current commit
+            - heads-cross-musl-{{ checksum "/tmp/musl-cross_module_and_patches.sha256sums" }}{{ .Environment.CACHE_VERSION }}
+
+# linuxboot steps need something to pass in the kernel header path
+# skipping for now
+#      - run:
+#          name: qemu-linuxboot-edk2
+#          command: |
+#            ./build/make-4.2.1/make \
+#                CROSS=/cross/bin/x86_64-linux-musl- \
+#                BOARD=qemu-linuxboot \
+#                `/bin/pwd`/build/linuxboot-git/build/qemu/.configured \
+#                # Run first to avoid too many processes
+#
+#      - run:
+#          name: qemu-linuxboot
+#          command: |
+#            ./build/make-4.2.1/make \
+#                CROSS=/cross/bin/x86_64-linux-musl- \
+#                --load 2 \
+#                V=1 \
+#                BOARD=qemu-linuxboot \
+#
+#      - store-artifacts:
+#          path: build/qemu-linuxboot/linuxboot.rom
+#      - store-artifacts:
+#          path: build/qemu-linuxboot/hashes.txt
 
       - run:
-          name: qemu-linuxboot
+          name: x230-flash
+          #We delete build/make-4.2.1/ directory until issue #799 is fixed.
           command: |
-            ./build/make-4.2.1/make \
-                TOOLCHAIN=/home/builder/heads \
+            rm -rf build/x230-flash/* build/log/* && make --load 2 \
                 V=1 \
-                -j4 \
-                BOARD=qemu-linuxboot \
-
+                BOARD=x230-flash || (find ./build/log/ -cmin 1|xargs tail; exit 1) \
+          no_output_timeout: 3h
+      - run:
+          name: Ouput x230-flash hashes
+          command: |
+            cat build/x230-flash/hashes.txt \
+      - run:
+          name: Archiving build logs for x230-flash
+          command: |
+             tar zcvf build/x230-flash/logs.tar.gz build/log/*
       - store-artifacts:
-          path: build/qemu-linuxboot/linuxboot.rom
+          path: build/x230-flash
+
+      - run:
+          name: t430-flash
+          command: |
+            rm -rf build/t430-flash/* build/log/* && make --load 2 \
+                V=1 \
+                BOARD=t430-flash || (find ./build/log/ -cmin 1|xargs tail; exit 1) \
+          no_output_timeout: 3h
+      - run:
+          name: Ouput t430-flash hashes
+          command: |
+            cat build/t430-flash/hashes.txt \
+      - run:
+          name: Archiving build logs for t430-flash
+          command: |
+             tar zcvf build/t430-flash/logs.tar.gz build/log/*
       - store-artifacts:
-          path: build/qemu-linuxboot/hashes.txt
+          path: build/t430-flash
 
+      - run:
+          name: t430
+          command: |
+            rm -rf build/t430/* build/log/* && make --load 2 \
+                V=1 \
+                BOARD=t430  || (find ./build/log/ -cmin 1|xargs tail; exit 1) \
+          no_output_timeout: 3h
+      - run:
+          name: Ouput t430 hashes
+          command: |
+            cat build/t430/hashes.txt \
+      - run:
+          name: Archiving build logs for t430
+          command: |
+             tar zcvf build/t430/logs.tar.gz build/log/*
+      - store-artifacts:
+          path: build/t430
 
       - run:
           name: x230
           command: |
-            ./build/make-4.2.1/make \
-                TOOLCHAIN=/home/builder/heads \
-                -j4 \
+            rm -rf build/x230/* build/log/* && make --load 2 \
                 V=1 \
-                BOARD=x230 \
+                BOARD=x230 || (find ./build/log/ -cmin 1|xargs tail; exit 1) \
+          no_output_timeout: 3h
+      - run:
+          name: Ouput x230 hashes
+          command: |
+            cat build/x230/hashes.txt \
+      - run:
+          name: Archiving build logs for x230
+          command: |
+             tar zcvf build/x230/logs.tar.gz build/log/*
+      - store-artifacts:
+          path: build/x230
 
+      - run:
+          name: x230-hotp-verification
+          command: |
+            rm -rf build/x230-hotp-verification/* build/log/* && make --load 2 \
+                V=1 \
+                BOARD=x230-hotp-verification || (find ./build/log/ -cmin 1|xargs tail; exit 1) \
+          no_output_timeout: 3h
+      - run:
+          name: Ouput x230-hotp-verification hashes
+          command: |
+            cat build/x230-hotp-verification/hashes.txt \
+      - run:
+          name: Archiving build logs for x230-hotp-verification
+          command: |
+             tar zcvf build/x230-hotp-verification/logs.tar.gz build/log/*
       - store-artifacts:
-          path: build/x230/coreboot.rom
+          path: build/x230-hotp-verification
+
+      - run:
+          name: qemu-coreboot
+          command: |
+            rm -rf build/qemu-coreboot/* build/log/* && make --load 2 \
+                V=1 \
+                BOARD=qemu-coreboot || (find ./build/log/ -cmin 1|xargs tail; exit 1) \
+          no_output_timeout: 3h
+      - run:
+          name: Output qemu-coreboot hashes
+          command: |
+             cat build/qemu-coreboot/hashes.txt \
+      - run:
+          name: Archiving build logs for qemu-coreboot
+          command: |
+             tar zcvf build/qemu-coreboot/logs.tar.gz build/log/*
       - store-artifacts:
-          path: build/x230/hashes.txt
+          path: build/qemu-coreboot
 
+      - save_cache:
+          #Generate cache for the same musl-cross module definition if hash is not previously existing
+          key: heads-cross-musl-{{ checksum "/tmp/musl-cross_module_and_patches.sha256sums" }}{{ .Environment.CACHE_VERSION }}
+          paths:
+            - crossgcc
+            - build/musl-cross-*
 
+      - save_cache:
+          #Generate cache for the exact same modules definitions if hash is not previously existing
+          key: heads-modules-and-patches-{{ checksum "/tmp/all_modules_and_patches.sha256sums" }}{{ .Environment.CACHE_VERSION }}
+          paths:
+            - packages
+            - crossgcc
+            - build
 workflows:
   version: 2
   build_and_test:

.gitlab-ci.yml

@@ -0,0 +1,87 @@
+image: fedora:30
+
+variables:
+  DOCKER_DRIVER: overlay2
+
+stages:
+  - build
+
+build:
+  stage: build
+  retry: 1
+  cache:
+    paths:
+      - packages
+      - crossgcc
+      - build
+    key: "heads-$GITLAB_USER_LOGIN-2"
+  script:
+    - dnf install -y @development-tools gcc-c++ gcc-gnat zlib-devel perl-Digest-MD5 perl-Digest-SHA uuid-devel pcsc-tools ncurses-devel lbzip2 libuuid-devel lzma elfutils-libelf-devel bc bzip2 bison flex git gnupg iasl m4 nasm patch python wget libusb-devel cmake automake pv bsdiff autoconf libtool cpio texinfo
+    - git fetch origin 
+    - git reset --hard origin/$CI_COMMIT_REF_NAME
+    - echo "Removing old x230-flash artifacts..."
+    - rm -rf ./build/x230-flash/*
+    - rm -rf ./build/log/*
+    - echo "Building BOARD=x230-flash board..."
+    - make BOARD=x230-flash || (find ./build/log/ -cmin 1|xargs tail; exit 1)
+    - echo "x230-flash hashes:"
+    - cat ./build/x230-flash/hashes.txt
+    - echo "Archiving x230-flash logs..."
+    - tar zcvf ./build/x230-flash/logs.tar.gz ./build/log/*
+    - echo "Removing old t430-flash artifacts..."
+    - rm -rf ./build/t430-flash/*
+    - rm -rf ./build/log/*
+    - echo "Building BOARD=t430-flash board..."
+    - make BOARD=t430-flash || (find ./build/log/ -cmin 1|xargs tail; exit 1)
+    - echo "t430-flash hashes:"
+    - cat ./build/t430-flash/hashes.txt
+    - echo "Archiving t430-flash logs..."
+    - tar zcvf ./build/t430-flash/logs.tar.gz ./build/log/*
+    - echo "Removing old x230-hotp-verification artifacts..."
+    - rm -rf ./build/x230-hotp-verification/*
+    - rm -rf ./build/log/*
+    - echo "Building BOARD=x230-hotp-verification board..."
+    - make BOARD=x230-hotp-verification || (find ./build/log/ -cmin 1|xargs tail; exit 1)
+    - echo "x230-hotp-verification hashes:"
+    - cat ./build/x230-hotp-verification/hashes.txt
+    - echo "Archiving x230-hotp-verification logs..."
+    - tar zcvf ./build/x230-hotp-verification/logs.tar.gz ./build/log/*
+    - echo "Removing old x230 artifacts..."
+    - rm -rf ./build/x230/*
+    - rm -rf ./build/log/*
+    - echo "Building BOARD=x230 board..."
+    - make BOARD=x230 || (find ./build/log/ -cmin 1|xargs tail; exit 1)
+    - echo "x230 hashes:"
+    - cat ./build/x230/hashes.txt
+    - echo "Archiving x230 logs..."
+    - tar zcvf ./build/x230/logs.tar.gz ./build/log/*
+
+    - echo "Removing old t430 artifacts..."
+    - rm -rf ./build/t430/*
+    - rm -rf ./build/log/*
+    - echo "Building BOARD=t430 board..."
+    - make BOARD=t430 || (find ./build/log/ -cmin 1|xargs tail; exit 1)
+    - echo "t430 hashes:"
+    - cat ./build/t430/hashes.txt
+    - echo "Archiving t430 logs..."
+    - tar zcvf ./build/t430/logs.tar.gz ./build/log/*
+
+    - echo "Removing old qemu-coreboot artifacts..."
+    - rm -rf ./build/qemu-coreboot/*
+    - rm -rf ./build/log/*
+    - echo "Building BOARD=qemu-coreboot board..."
+    - make BOARD=qemu-coreboot || (find ./build/log/ -cmin 1|xargs tail; exit 1)
+    - echo "qemu-coreboot hashes:"
+    - cat ./build/qemu-coreboot/hashes.txt
+    - echo "Archiving qemu-coreboot logs..."
+    - tar zcvf ./build/qemu-coreboot/logs.tar.gz ./build/log/*
+    - echo "Calculate used space for cache"
+    - du -shc packages crossgcc build
+  artifacts:
+    paths:
+      - ./build/qemu-coreboot
+      - ./build/x230-flash
+      - ./build/t430-flash
+      - ./build/x230-hotp-verification
+      - ./build/x230
+      - ./build/t430