From 073695d07151238cd9e23f74c3a7f47b468b0388 Mon Sep 17 00:00:00 2001 From: Suraj Jacob <28795567+jacobmsft@users.noreply.github.com> Date: Tue, 14 Mar 2023 14:36:31 -0700 Subject: [PATCH] Upgrades (#51) increment ubuntu container version move from pygithub to ghapi skip the chown step --- Dockerfile | 66 ++++++++++++-------------- container/get-latest-codeql-version.py | 8 +--- container/libs/codeql.py | 8 ++-- container/libs/github.py | 18 +++---- container/requirements.txt | 3 +- container/setup.py | 2 +- container/startup.py | 1 - 7 files changed, 50 insertions(+), 56 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6313e14..3022669 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 AS codeql_base +FROM ubuntu:22.04 AS codeql_base LABEL maintainer="Github codeql team" # tzdata install needs to be non-interactive @@ -12,33 +12,31 @@ RUN adduser --home ${CODEQL_HOME} ${USERNAME} && \ apt-get update && \ apt-get upgrade -y && \ apt-get install -y --no-install-recommends \ - software-properties-common \ - nodejs \ - vim \ - curl \ - wget \ - git \ - build-essential \ - unzip \ - apt-transport-https \ - python3.8 \ - python3-venv \ - python3-pip \ - python3-setuptools \ - python3-dev \ - gnupg \ - g++ \ - make \ - gcc \ - apt-utils \ - rsync \ - file \ - dos2unix \ - gettext && \ - apt-get clean && \ - rm -f /usr/bin/python /usr/bin/pip && \ - ln -s /usr/bin/python3.8 /usr/bin/python && \ - ln -s /usr/bin/pip3 /usr/bin/pip + software-properties-common \ + nodejs \ + vim \ + curl \ + wget \ + git \ + build-essential \ + unzip \ + apt-transport-https \ + python3.10 \ + python3-venv \ + python3-pip \ + python3-setuptools \ + python3-dev \ + python-is-python3 \ + gnupg \ + g++ \ + make \ + gcc \ + apt-utils \ + rsync \ + file \ + dos2unix \ + gettext && \ + apt-get clean # Install .NET Core and Java for tools/builds RUN cd /tmp && \ @@ -48,15 +46,13 @@ RUN cd /tmp && \ apt-get install -y default-jdk apt-transport-https && \ apt-get update && \ rm packages-microsoft-prod.deb -RUN apt-get install -y dotnet-sdk-3.1 +RUN apt-get install -y dotnet-sdk-6.0 # Clone our setup and run scripts -#RUN git clone https://github.com/microsoft/codeql-container /usr/local/startup_scripts RUN mkdir -p /usr/local/startup_scripts -RUN ls -al /usr/local/startup_scripts COPY container /usr/local/startup_scripts/ -RUN pip3 install --upgrade pip \ - && pip3 install -r /usr/local/startup_scripts/requirements.txt + +RUN pip3 install -r /usr/local/startup_scripts/requirements.txt # Install latest codeQL @@ -83,8 +79,8 @@ RUN codeql query compile --threads=0 ${CODEQL_HOME}/codeql-repo/*/ql/src/codeql- ENV PYTHONIOENCODING=utf-8 # Change ownership of all files and directories within CODEQL_HOME to the codeql user -RUN chown -R ${USERNAME}:${USERNAME} ${CODEQL_HOME} +#RUN chown -R ${USERNAME}:${USERNAME} ${CODEQL_HOME} USER ${USERNAME} -ENTRYPOINT ["python3", "/usr/local/startup_scripts/startup.py"] +ENTRYPOINT ["python3", "/usr/local/startup_scripts/startup.py"] \ No newline at end of file diff --git a/container/get-latest-codeql-version.py b/container/get-latest-codeql-version.py index d2972bd..8229c05 100644 --- a/container/get-latest-codeql-version.py +++ b/container/get-latest-codeql-version.py @@ -1,13 +1,9 @@ #!/usr/bin/env python3 # get the parent directory of the script, to link libs - -import os -import sys - from libs.github import get_latest_github_repo_version def main(): - latest_release = get_latest_github_repo_version("github/codeql-cli-binaries") - print(latest_release.title) + latest_release = get_latest_github_repo_version("github", "codeql-cli-binaries") + print(latest_release.tag_name) main() diff --git a/container/libs/codeql.py b/container/libs/codeql.py index 7b6fba0..006c3bf 100644 --- a/container/libs/codeql.py +++ b/container/libs/codeql.py @@ -34,15 +34,15 @@ def download_and_install_latest_codeql(self, github_version): download_url = None download_path = None if os_name == 'posix': - download_url = f'{self.CODEQL_GITHUB_URL}/releases/download/{github_version.title}/codeql-linux64.zip' + download_url = f'{self.CODEQL_GITHUB_URL}/releases/download/{github_version.tag_name}/codeql-linux64.zip' download_path = f'{self.TEMP_DIR}/codeql_linux.zip' elif os_name == 'nt': - download_url = f'{self.CODEQL_GITHUB_URL}/releases/download/{github_version.title}/codeql-win64.zip' + download_url = f'{self.CODEQL_GITHUB_URL}/releases/download/{github_version.tag_name}/codeql-win64.zip' download_path = f'{self.TEMP_DIR}/codeql_windows.zip' else: exit(self.ERROR_UNKNOWN_OS) - logger.info(f'Downloading codeql-cli version {github_version.title}...') + logger.info(f'Downloading codeql-cli version {github_version.tag_name}...') check_output_wrapper(f"wget -q {download_url} -O {download_path}", shell=True).decode("utf-8") self.install_codeql_cli(download_path) #rm /tmp/codeql_linux.zip @@ -77,7 +77,7 @@ def get_current_local_version(self): return version def get_latest_codeql_github_version(self): - return get_latest_github_repo_version("github/codeql-cli-binaries") + return get_latest_github_repo_version("github", "codeql-cli-binaries") def install_codeql_cli(self, download_path): logger.info("Installing codeql-cli...") diff --git a/container/libs/github.py b/container/libs/github.py index d576216..cd3cfd6 100644 --- a/container/libs/github.py +++ b/container/libs/github.py @@ -1,24 +1,26 @@ import os from datetime import datetime, MINYEAR -from github import Github, GitRelease, Repository, GithubException +from ghapi.all import GhApi +from datetime import datetime, timezone +from dateutil import parser -def get_latest_github_repo_version(repo): +def get_latest_github_repo_version(owner, repository): # check for a github token that may be used alongside the codeql cli to upload github results # this will limit rate limting 403 errors on checking codeql versions, as the request will be authenticated if possible. # by default codeql uses env var "GITHUB_TOKEN" to authenticate # https://codeql.github.com/docs/codeql-cli/manual/github-upload-results/ access_token = os.getenv('GITHUB_TOKEN') - client = Github(access_token) if access_token != None else Github() - repo = client.get_repo(repo) - releases = repo.get_releases() + api = GhApi(owner=owner, repo=repository, token=access_token) if access_token != None else GhApi(owner=owner, repo=repository) + releases = api.repos.list_releases() latest_release = get_latest_github_release(releases) return latest_release def get_latest_github_release(releases): latest_release = None - latest_date = datetime(MINYEAR, 1, 1) + latest_date = datetime(MINYEAR, 1, 1).replace(tzinfo=timezone.utc) for release in releases: - if release.created_at > latest_date: - latest_date = release.created_at + release_date = parser.parse(release.created_at) + if release_date > latest_date: + latest_date = release_date latest_release = release return latest_release diff --git a/container/requirements.txt b/container/requirements.txt index f15ff22..ce3acfc 100644 --- a/container/requirements.txt +++ b/container/requirements.txt @@ -1 +1,2 @@ -PyGithub==1.43.7 \ No newline at end of file +ghapi==1.0.3 +python-dateutil==2.8.2 \ No newline at end of file diff --git a/container/setup.py b/container/setup.py index e3fc8d1..56a148f 100644 --- a/container/setup.py +++ b/container/setup.py @@ -46,7 +46,7 @@ def get_latest_codeql(args): # ensure we only query for the latest codeql cli version if we might actually update it if args.check_latest_cli: latest_online_version = codeql.get_latest_codeql_github_version() - if current_installed_version != latest_online_version.title: + if current_installed_version != latest_online_version.tag_name: # we got a newer version online, download and install it codeql.download_and_install_latest_codeql(latest_online_version) # get the latest queries regardless (TODO: Optimize by storing and checking the last commit hash?) diff --git a/container/startup.py b/container/startup.py index 94319d7..dc05277 100755 --- a/container/startup.py +++ b/container/startup.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 import os -import sys from time import sleep from libs.utils import get_env_variable, check_output_wrapper, get_logger from libs.codeql import *