From f8666d40738fe515ed1ef51f3b9835f0247dac01 Mon Sep 17 00:00:00 2001 From: Michael Sprengel Date: Thu, 21 Dec 2023 14:55:25 +0100 Subject: [PATCH] Add data ingestion mechanism (#2) --- .github/workflows/data_ingestion.yml | 119 +++++++++++++++++++++++++-- .gitignore | 1 + conf/ingest-debsrc/apt.conf | 5 ++ conf/ingest-debsrc/debian.sources | 6 ++ data/CVE/list | 2 + src/prepare_ingestion.sh | 24 ++++++ 6 files changed, 150 insertions(+), 7 deletions(-) create mode 100644 .gitignore create mode 100644 conf/ingest-debsrc/apt.conf create mode 100644 conf/ingest-debsrc/debian.sources create mode 100644 data/CVE/list create mode 100755 src/prepare_ingestion.sh diff --git a/.github/workflows/data_ingestion.yml b/.github/workflows/data_ingestion.yml index 12ef536..4f5ce14 100644 --- a/.github/workflows/data_ingestion.yml +++ b/.github/workflows/data_ingestion.yml @@ -1,13 +1,118 @@ name: Data ingestion -on: ["push", "workflow_dispatch"] +on: + push: + workflow_dispatch: + schedule: + - cron: '0 2 * * *' jobs: - test: + data_ingestion: environment: glvd-data-ingestion name: data ingestion runs-on: ubuntu-latest + + env: + PGHOST: "${{ vars.PGHOST }}" + PGPORT: "${{ vars.PGPORT }}" + PGUSER: "${{ vars.PGUSER }}" + PGPASSWORD: "${{ secrets.PGPASSWORD }}" + PGDATABASE: "${{ vars.PGDATABASE }}" + steps: - - run: | - echo "${{ vars.PGDATABASE }}" - echo "${{ vars.PGHOST }}" - echo "${{ vars.PGPORT }}" - echo "${{ vars.PGUSER }}" \ No newline at end of file + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Prepare data ingestion + run: ./src/prepare_ingestion.sh + + - name: Run data ingestion (ingest-debsec - debian) + run: > + docker run + --network host + --rm + --volume $PWD/tmp/:/opt/glvd-data-ingestion + --entrypoint="" + --env PGHOST + --env PGPORT + --env PGUSER + --env PGPASSWORD + --env PGDATABASE + ghcr.io/gardenlinux/glvd:edge + /bin/sh -c 'glvd-data ingest-debsec debian /opt/glvd-data-ingestion/ingest-debsec/debian' + + - name: Run data ingestion (ingest-debsec - gardenlinux) + run: > + docker run + --network host + --rm + --volume $PWD/tmp/:/opt/glvd-data-ingestion + --entrypoint="" + --env PGHOST + --env PGPORT + --env PGUSER + --env PGPASSWORD + --env PGDATABASE + ghcr.io/gardenlinux/glvd:edge + /bin/sh -c 'glvd-data ingest-debsec gardenlinux /opt/glvd-data-ingestion/ingest-debsec/gardenlinux' + + - name: Run data ingestion (ingest-debsrc - debian buster) + run: > + docker run + --network host + --rm + --volume $PWD/tmp/:/opt/glvd-data-ingestion + --entrypoint="" + --env PGHOST + --env PGPORT + --env PGUSER + --env PGPASSWORD + --env PGDATABASE + ghcr.io/gardenlinux/glvd:edge + /bin/sh -c 'glvd-data ingest-debsrc debian buster /opt/glvd-data-ingestion/ingest-debsrc/lists/deb.debian.org_debian_dists_buster_main_source_Sources' + + - name: Run data ingestion (ingest-debsrc - debian bullseye) + run: > + docker run + --network host + --rm + --volume $PWD/tmp/:/opt/glvd-data-ingestion + --entrypoint="" + --env PGHOST + --env PGPORT + --env PGUSER + --env PGPASSWORD + --env PGDATABASE + ghcr.io/gardenlinux/glvd:edge + /bin/sh -c 'glvd-data ingest-debsrc debian bullseye /opt/glvd-data-ingestion/ingest-debsrc/lists/deb.debian.org_debian_dists_bullseye_main_source_Sources' + + - name: Run data ingestion (ingest-debsrc - debian bookworm) + run: > + docker run + --network host + --rm + --volume $PWD/tmp/:/opt/glvd-data-ingestion + --entrypoint="" + --env PGHOST + --env PGPORT + --env PGUSER + --env PGPASSWORD + --env PGDATABASE + ghcr.io/gardenlinux/glvd:edge + /bin/sh -c 'glvd-data ingest-debsrc debian bookworm /opt/glvd-data-ingestion/ingest-debsrc/lists/deb.debian.org_debian_dists_bookworm_main_source_Sources' + + - name: Run data ingestion (ingest-debsrc - debian trixie) + run: > + docker run + --network host + --rm + --volume $PWD/tmp/:/opt/glvd-data-ingestion + --entrypoint="" + --env PGHOST + --env PGPORT + --env PGUSER + --env PGPASSWORD + --env PGDATABASE + ghcr.io/gardenlinux/glvd:edge + /bin/sh -c 'glvd-data ingest-debsrc debian trixie /opt/glvd-data-ingestion/ingest-debsrc/lists/deb.debian.org_debian_dists_trixie_main_source_Sources' diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1fb9ef5 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +tmp/* diff --git a/conf/ingest-debsrc/apt.conf b/conf/ingest-debsrc/apt.conf new file mode 100644 index 0000000..7e0f5f6 --- /dev/null +++ b/conf/ingest-debsrc/apt.conf @@ -0,0 +1,5 @@ +Acquire::Retries "3"; +Dir::Etc::parts "/dev/null"; +Dir::Etc::Preferences "/dev/null"; +Dir::Etc::PreferencesParts "/dev/null"; +Dir::Etc::sourceparts "/dev/null"; diff --git a/conf/ingest-debsrc/debian.sources b/conf/ingest-debsrc/debian.sources new file mode 100644 index 0000000..7c7024a --- /dev/null +++ b/conf/ingest-debsrc/debian.sources @@ -0,0 +1,6 @@ +Enabled: yes +Types: deb-src +URIs: https://deb.debian.org/debian +Suites: buster bullseye bookworm trixie +Components: main +Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg diff --git a/data/CVE/list b/data/CVE/list new file mode 100644 index 0000000..8562e38 --- /dev/null +++ b/data/CVE/list @@ -0,0 +1,2 @@ +CVE-2023-6977 (This vulnerability enables malicious users to read sensitive files on ...) + TODO: check diff --git a/src/prepare_ingestion.sh b/src/prepare_ingestion.sh new file mode 100755 index 0000000..e0df452 --- /dev/null +++ b/src/prepare_ingestion.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -e + +# Install Debian Keyring +sudo apt update +sudo apt install -y debian-archive-keyring + +# Prepare: mount directories +mkdir -p tmp/ingest-debsec/{debian,gardenlinux}/CVE +mkdir -p tmp/ingest-debsrc/debian +mkdir -p tmp/ingest-debsrc/var/lib/dpkg +touch tmp/ingest-debsrc/var/lib/dpkg/status + +# Prepare: ingest-debsec +curl https://salsa.debian.org/security-tracker-team/security-tracker/-/raw/master/data/CVE/list?ref_type=heads \ + --output tmp/ingest-debsec/debian/CVE/list +cp -p data/CVE/list tmp/ingest-debsec/gardenlinux/CVE/list + +# Prepare: ingest-debsrc +APT_CONFIG=conf/ingest-debsrc/apt.conf apt update \ + -o Dir="$PWD/tmp/ingest-debsrc/" \ + -o Dir::Etc::sourcelist="$PWD/conf/ingest-debsrc/debian.sources" \ + -o Dir::State="$PWD/tmp/ingest-debsrc/"