From c4e75ee89cdd18e181fd78f8b8e1ed5af3fe8fc0 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sun, 18 Aug 2024 11:40:54 -0700 Subject: [PATCH] remove cereal dependence (#2000) * remove cereal * rm more shit * add back make * python3 * python3 * fix that * put that back * rename * fix that * can't move it --- .github/workflows/test.yaml | 8 +++--- Dockerfile | 50 +++++++++++-------------------------- Jenkinsfile | 4 +-- tests/misra/test_misra.sh | 2 +- tests/usbprotocol/test.sh | 2 +- 5 files changed, 22 insertions(+), 44 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 97c9e07f0e..5b62bf18fd 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -11,7 +11,7 @@ concurrency: cancel-in-progress: true env: - RUN: docker run -v ${{ github.workspace }}:/tmp/openpilot/panda -w /tmp/openpilot/panda --rm panda /bin/bash -c + RUN: docker run -v ${{ github.workspace }}:/tmp/pythonpath/panda -w /tmp/pythonpath/panda --rm panda /bin/bash -c BUILD: | export DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from ghcr.io/commaai/panda:latest -t panda -f Dockerfile . @@ -45,7 +45,7 @@ jobs: - name: Build Docker image run: eval "$BUILD" - name: Test python package installer - run: ${{ env.RUN }} "python setup.py install" + run: ${{ env.RUN }} "python3 setup.py install" - name: Build panda images and bootstub run: ${{ env.RUN }} "scons -j4" - name: Build panda with SPI support @@ -103,8 +103,8 @@ jobs: timeout-minutes: 5 run: ${{ env.RUN }} "cd tests/misra && pytest -n8 test_mutation.py" - python_linter: - name: python linter + static_analysis: + name: static analysis runs-on: ubuntu-latest timeout-minutes: 20 steps: diff --git a/Dockerfile b/Dockerfile index 1a6122ee05..51a58ca803 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,44 +1,25 @@ FROM ubuntu:24.04 -ENV PYTHONUNBUFFERED 1 -ENV PYTHONPATH /tmp/openpilot:$PYTHONPATH +ENV PYTHONUNBUFFERED=1 +ENV PYTHONPATH=/tmp/pythonpath ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ make \ - bzip2 \ - ca-certificates \ - capnproto \ - clang \ g++ \ gcc-arm-none-eabi libnewlib-arm-none-eabi \ git \ - libarchive-dev \ - libbz2-dev \ - libcapnp-dev \ libffi-dev \ - libtool \ libusb-1.0-0 \ - libzmq3-dev \ - locales \ - opencl-headers \ - ocl-icd-opencl-dev \ python3 \ python3-dev \ python3-pip \ - python-is-python3 \ - zlib1g-dev \ && rm -rf /var/lib/apt/lists/* && \ apt clean && \ cd /usr/lib/gcc/arm-none-eabi/* && \ rm -rf arm/ && \ rm -rf thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp -RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 - COPY requirements.txt /tmp/ RUN pip3 install --break-system-packages --no-cache-dir -r /tmp/requirements.txt @@ -47,21 +28,18 @@ COPY tests/misra/install.sh /tmp/ RUN /tmp/install.sh && rm -rf $CPPCHECK_DIR/.git/ ENV SKIP_CPPCHECK_INSTALL=1 -ENV CEREAL_REF="861144c136c91f70dcbc652c2ffe99f57440ad47" -ENV OPENDBC_REF="8e9d3688412405154a8189c421cfdc9d5feea715" - -RUN git config --global --add safe.directory /tmp/openpilot/panda -RUN mkdir -p /tmp/openpilot/ && \ - cd /tmp/openpilot/ && \ - git clone --depth 1 https://github.com/commaai/cereal && \ - git clone --depth 1 https://github.com/commaai/opendbc && \ - cd cereal && git fetch origin $CEREAL_REF && git checkout FETCH_HEAD && rm -rf .git/ && cd .. && \ - cd opendbc && git fetch origin $OPENDBC_REF && git checkout FETCH_HEAD && rm -rf .git/ && cd .. && \ - cp -pR opendbc/SConstruct opendbc/site_scons/ . && \ - pip3 install --break-system-packages --no-cache-dir -r opendbc/requirements.txt && \ - scons -j8 --minimal opendbc/ cereal/ +# TODO: this should be a "pip install" or not even in this repo at all +RUN git config --global --add safe.directory $PYTHONPATH/panda +ENV OPENDBC_REF="5ed7a834a4e0e24c3968dd1e98ceb4b9d5f9791a" +RUN mkdir -p $PYTHONPATH && \ + cd /tmp/ && \ + git clone --depth 1 https://github.com/commaai/opendbc opendbc_repo && \ + cd opendbc_repo && git fetch origin $OPENDBC_REF && git checkout FETCH_HEAD && rm -rf .git/ && \ + pip3 install --break-system-packages --no-cache-dir Cython numpy && \ + scons -j8 --minimal opendbc/ && \ + ln -s $PWD/opendbc $PYTHONPATH/opendbc # for Jenkins COPY README.md panda.tar.* /tmp/ -RUN mkdir /tmp/openpilot/panda && \ - tar -xvf /tmp/panda.tar.gz -C /tmp/openpilot/panda/ || true +RUN mkdir /tmp/pythonpath/panda && \ + tar -xvf /tmp/panda.tar.gz -C /tmp/pythonpath/panda/ || true diff --git a/Jenkinsfile b/Jenkinsfile index b3b2690861..2ccf6be116 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,7 +4,7 @@ def docker_run(String step_label, int timeout_mins, String cmd) { --env PYTHONWARNINGS=error \ --volume /dev/bus/usb:/dev/bus/usb \ --volume /var/run/dbus:/var/run/dbus \ - --workdir /tmp/openpilot/panda \ + --workdir /tmp/pythonpath/panda \ --net host \ ${env.DOCKER_IMAGE_TAG} \ bash -c 'scons -j8 && ${cmd}'", \ @@ -92,7 +92,7 @@ pipeline { steps { script { retry (3) { - docker_run("reset hardware", 3, "python ./tests/hitl/reset_jungles.py") + docker_run("reset hardware", 3, "python3 ./tests/hitl/reset_jungles.py") } } } diff --git a/tests/misra/test_misra.sh b/tests/misra/test_misra.sh index e65f6a5e5a..7fd110467c 100755 --- a/tests/misra/test_misra.sh +++ b/tests/misra/test_misra.sh @@ -19,7 +19,7 @@ fi # ensure checked in coverage table is up to date cd $DIR if [ -z "$SKIP_TABLES_DIFF" ]; then - python $CPPCHECK_DIR/addons/misra.py -generate-table > coverage_table + python3 $CPPCHECK_DIR/addons/misra.py -generate-table > coverage_table if ! git diff --quiet coverage_table; then echo -e "${YELLOW}MISRA coverage table doesn't match. Update and commit:${NC}" exit 3 diff --git a/tests/usbprotocol/test.sh b/tests/usbprotocol/test.sh index 8e3886da7d..b4c32166b9 100755 --- a/tests/usbprotocol/test.sh +++ b/tests/usbprotocol/test.sh @@ -4,5 +4,5 @@ set -e # Loops over all HW_TYPEs, see board/boards/board_declarations.h for hw_type in {0..7}; do echo "Testing HW_TYPE: $hw_type" - HW_TYPE=$hw_type python -m unittest discover . + HW_TYPE=$hw_type python3 -m unittest discover . done