Add data ingestion mechanism

gardenlinux · Dec 21, 2023 · 4028939 · 4028939
1 parent cf07f7c
commit 4028939
Show file tree

Hide file tree

Showing 6 changed files with 149 additions and 8 deletions.
diff --git a/.github/workflows/data_ingestion.yml b/.github/workflows/data_ingestion.yml
@@ -1,13 +1,118 @@
 name: Data ingestion
-on: ["push", "workflow_dispatch"]
+on: 
+  push:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 2 * * *'
 jobs:
-  test:
+  data_ingestion:
     environment: glvd-data-ingestion
     name: data ingestion
     runs-on: ubuntu-latest
+
+    env:
+      PGHOST: "${{ vars.PGHOST }}"
+      PGPORT: "${{ vars.PGPORT }}"
+      PGUSER: "${{ vars.PGUSER }}"
+      PGPASSWORD: "${{ secrets.PGPASSWORD }}"
+      PGDATABASE: "${{ vars.PGDATABASE }}"
+
     steps:
-      - run: |
-          echo "${{ vars.PGDATABASE }}"
-          echo "${{ vars.PGHOST }}"
-          echo "${{ vars.PGPORT }}"
-          echo "${{ vars.PGUSER }}"
+    - name: Checkout repo
+      uses: actions/checkout@v4
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+
+    - name: Prepare data ingestion
+      run: ./src/prepare_ingestion.sh
+
+    - name: Run data ingestion (ingest-debsec - debian)
+      run: >
+        docker run
+        --network host
+        --rm
+        --volume $PWD/tmp/:/opt/glvd-data-ingestion
+        --entrypoint=""
+        --env PGHOST
+        --env PGPORT
+        --env PGUSER
+        --env PGPASSWORD
+        --env PGDATABASE
+        ghcr.io/gardenlinux/glvd:edge
+        /bin/sh -c 'glvd-data ingest-debsec debian /opt/glvd-data-ingestion/ingest-debsec/debian'
+
+    - name: Run data ingestion (ingest-debsec - gardenlinux)
+      run: >
+        docker run
+        --network host
+        --rm
+        --volume $PWD/tmp/:/opt/glvd-data-ingestion
+        --entrypoint=""
+        --env PGHOST
+        --env PGPORT
+        --env PGUSER
+        --env PGPASSWORD
+        --env PGDATABASE
+        ghcr.io/gardenlinux/glvd:edge
+        /bin/sh -c 'glvd-data ingest-debsec gardenlinux /opt/glvd-data-ingestion/ingest-debsec/gardenlinux'
+
+    - name: Run data ingestion (ingest-debsrc - debian buster)
+      run: >
+        docker run
+        --network host
+        --rm
+        --volume $PWD/tmp/:/opt/glvd-data-ingestion
+        --entrypoint=""
+        --env PGHOST
+        --env PGPORT
+        --env PGUSER
+        --env PGPASSWORD
+        --env PGDATABASE
+        ghcr.io/gardenlinux/glvd:edge
+        /bin/sh -c 'glvd-data ingest-debsrc debian buster /opt/glvd-data-ingestion/ingest-debsrc/lists/deb.debian.org_debian_dists_buster_main_source_Sources'
+
+    - name: Run data ingestion (ingest-debsrc - debian bullseye)
+      run: >
+        docker run
+        --network host
+        --rm
+        --volume $PWD/tmp/:/opt/glvd-data-ingestion
+        --entrypoint=""
+        --env PGHOST
+        --env PGPORT
+        --env PGUSER
+        --env PGPASSWORD
+        --env PGDATABASE
+        ghcr.io/gardenlinux/glvd:edge
+        /bin/sh -c 'glvd-data ingest-debsrc debian bullseye /opt/glvd-data-ingestion/ingest-debsrc/lists/deb.debian.org_debian_dists_bullseye_main_source_Sources'
+
+    - name: Run data ingestion (ingest-debsrc - debian bookworm)
+      run: >
+        docker run
+        --network host
+        --rm
+        --volume $PWD/tmp/:/opt/glvd-data-ingestion
+        --entrypoint=""
+        --env PGHOST
+        --env PGPORT
+        --env PGUSER
+        --env PGPASSWORD
+        --env PGDATABASE
+        ghcr.io/gardenlinux/glvd:edge
+        /bin/sh -c 'glvd-data ingest-debsrc debian bookworm /opt/glvd-data-ingestion/ingest-debsrc/lists/deb.debian.org_debian_dists_bookworm_main_source_Sources'
+
+    - name: Run data ingestion (ingest-debsrc - debian trixie)
+      run: >
+        docker run
+        --network host
+        --rm
+        --volume $PWD/tmp/:/opt/glvd-data-ingestion
+        --entrypoint=""
+        --env PGHOST
+        --env PGPORT
+        --env PGUSER
+        --env PGPASSWORD
+        --env PGDATABASE
+        ghcr.io/gardenlinux/glvd:edge
+        /bin/sh -c 'glvd-data ingest-debsrc debian trixie /opt/glvd-data-ingestion/ingest-debsrc/lists/deb.debian.org_debian_dists_trixie_main_source_Sources'
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1 @@
+tmp/*
diff --git a/conf/ingest-debsrc/apt.conf b/conf/ingest-debsrc/apt.conf
@@ -0,0 +1,5 @@
+Acquire::Retries "3";
+Dir::Etc::parts "/dev/null";
+Dir::Etc::Preferences "/dev/null";
+Dir::Etc::PreferencesParts "/dev/null";
+Dir::Etc::sourceparts "/dev/null";
diff --git a/conf/ingest-debsrc/debian.sources b/conf/ingest-debsrc/debian.sources
@@ -0,0 +1,6 @@
+Enabled: yes
+Types: deb-src
+URIs: https://deb.debian.org/debian 
+Suites: buster bullseye bookworm trixie
+Components: main 
+Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg
diff --git a/data/CVE/list b/data/CVE/list
@@ -1,2 +1,2 @@
 CVE-2023-6977 (This vulnerability enables malicious users to read sensitive files on  ...)
-	TODO: check
+	TODO: check
diff --git a/src/prepare_ingestion.sh b/src/prepare_ingestion.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+# Install Debian Keyring
+sudo apt update
+sudo apt install -y debian-archive-keyring
+
+# Prepare: mount directories
+mkdir -p tmp/ingest-debsec/{debian,gardenlinux}/CVE
+mkdir -p tmp/ingest-debsrc/debian
+mkdir -p tmp/ingest-debsrc/var/lib/dpkg
+touch tmp/ingest-debsrc/var/lib/dpkg/status
+
+# Prepare: ingest-debsec
+curl https://salsa.debian.org/security-tracker-team/security-tracker/-/raw/master/data/CVE/list?ref_type=heads \
+    --output tmp/ingest-debsec/debian/CVE/list
+cp -p data/CVE/list tmp/ingest-debsec/gardenlinux/CVE/list
+
+# Prepare: ingest-debsrc
+APT_CONFIG=conf/ingest-debsrc/apt.conf apt update \
+  -o Dir="$PWD/tmp/ingest-debsrc/" \
+  -o Dir::Etc::sourcelist="$PWD/conf/ingest-debsrc/debian.sources" \
+  -o Dir::State="$PWD/tmp/ingest-debsrc/"