diff --git a/.github/jobs/build_sonarqube_image.sh b/.github/jobs/build_sonarqube_image.sh index 9cdeea3395..55d558624b 100755 --- a/.github/jobs/build_sonarqube_image.sh +++ b/.github/jobs/build_sonarqube_image.sh @@ -28,7 +28,6 @@ time_command docker build -t ${DOCKERHUB_TAG} \ --build-arg MET_BASE_REPO \ --build-arg MET_BASE_TAG \ --build-arg SOURCE_BRANCH \ - --build-arg MET_CONFIG_OPTS \ --build-arg SONAR_SCANNER_VERSION \ --build-arg SONAR_HOST_URL \ --build-arg SONAR_TOKEN \ @@ -41,6 +40,6 @@ fi # Copy the .scannerwork directory from the image id=$(docker create ${DOCKERHUB_TAG}) -time_command docker cp $id:/met/.scannerwork /tmp/met_scannerwork +time_command mkdir -p /tmp/scannerwork +time_command docker cp $id:/met/.scannerwork/report-task.txt /tmp/scannerwork/report-task.txt docker rm -v $id - diff --git a/.github/workflows/sonarqube.yml b/.github/workflows/sonarqube.yml index 36c2ccc02d..6a6627fb69 100644 --- a/.github/workflows/sonarqube.yml +++ b/.github/workflows/sonarqube.yml @@ -64,26 +64,15 @@ jobs: MET_BASE_TAG: v3.2 SOURCE_BRANCH: ${{ steps.get_branch_name.outputs.branch_name }} WD_REFERENCE_BRANCH: ${{ github.event.inputs.reference_branch }} - MET_CONFIG_OPTS: '--enable-all' SONAR_SCANNER_VERSION: 5.0.1.3006 SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - - name: SonarQube Python Quality Gate check - id: sonarqube-python-quality-gate-check + - name: SonarQube Quality Gate check + id: sonarqube-quality-gate-check uses: sonarsource/sonarqube-quality-gate-action@master with: - scanMetadataReportFile: /tmp/met_scannerwork/python-report-task.txt - timeout-minutes: 5 - env: - SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - - - name: SonarQube CXX Quality Gate check - id: sonarqube-cxx-quality-gate-check - uses: sonarsource/sonarqube-quality-gate-action@master - with: - scanMetadataReportFile: /tmp/met_scannerwork/cxx-report-task.txt + scanMetadataReportFile: /tmp/scannerwork/report-task.txt timeout-minutes: 5 env: SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} @@ -100,4 +89,3 @@ jobs: name: logs_sonarqube path: ${{ runner.workspace }}/logs if-no-files-found: ignore - diff --git a/internal/scripts/docker/Dockerfile.sonarqube b/internal/scripts/docker/Dockerfile.sonarqube index 156fa14bf7..75dc3c952c 100644 --- a/internal/scripts/docker/Dockerfile.sonarqube +++ b/internal/scripts/docker/Dockerfile.sonarqube @@ -92,4 +92,3 @@ RUN if [ ! -e "${MET_REPO_DIR}/configure.ac" ]; then \ RUN cd ${MET_REPO_DIR} \ && internal/scripts/docker/build_met_sonarqube.sh - diff --git a/internal/scripts/docker/build_met_sonarqube.sh b/internal/scripts/docker/build_met_sonarqube.sh index f726d8b3b9..45e1d10fc5 100755 --- a/internal/scripts/docker/build_met_sonarqube.sh +++ b/internal/scripts/docker/build_met_sonarqube.sh @@ -5,7 +5,7 @@ # # This build_met_sonarqube.sh script must be run from the top-level # directory of the MET repository to be analyzed. It runs SonarQube to -# scan both the Python and C/C++ MET source code. +# scan the MET source code. # # Usage: internal/scripts/docker/build_met_sonarqube.sh # @@ -48,12 +48,6 @@ if [ -z ${SONAR_REFERENCE_BRANCH+x} ]; then exit 1 fi -# Check whether MET_CONFIG_OPTS is defined -if [ -z ${MET_CONFIG_OPTS+x} ]; then - MET_CONFIG_OPTS='--enable-all' - echo "Setting MET_CONFIG_OPTS=${MET_CONFIG_OPTS} to scan all available options." -fi - # Locate the wrapper WRAPPER_NAME=build-wrapper-linux-x86-64 SONAR_WRAPPER=$(which $WRAPPER_NAME 2> /dev/null) @@ -81,52 +75,33 @@ if [ -z ${SONARQUBE_OUT_DIR} ]; then export SONARQUBE_OUT_DIR=bw-outputs fi +# Define the version string +SONAR_PROJECT_VERSION=$(cat docs/version | cut -d'=' -f2 | tr -d '" ') + # Store the full path to the scripts directory SONAR_PROPERTIES_DIR=internal/scripts/sonarqube SONAR_PROPERTIES=sonar-project.properties -# Copy sonar-project.properties for Python code +# Configure the sonar-project.properties [ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES -sed -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \ +sed -e "s|SONAR_PROJECT_KEY|MET-GHA|" \ + -e "s|SONAR_PROJECT_NAME|MET GHA|" \ + -e "s|SONAR_PROJECT_VERSION|$SONAR_PROJECT_VERSION|" \ -e "s|SONAR_HOST_URL|$SONAR_HOST_URL|" \ - -e "s|SONAR_PROJECT_KEY|MET-GHA-Python|" \ - -e "s|SONAR_PROJECT_NAME|MET GHA Python|" \ + -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \ -e "s|SONAR_BRANCH_NAME|$MET_GIT_NAME|" \ - -e "s|SONAR_REFERENCE_BRANCH|$SONAR_REFERENCE_BRANCH|" \ - $SONAR_PROPERTIES_DIR/python.sonar-project.properties > $SONAR_PROPERTIES + $SONAR_PROPERTIES_DIR/$SONAR_PROPERTIES > $SONAR_PROPERTIES # The source and reference branches must differ to define new code if [ "$MET_GIT_NAME" != "$SONAR_REFERENCE_BRANCH" ]; then echo "sonar.newCode.referenceBranch=${SONAR_REFERENCE_BRANCH}" >> $SONAR_PROPERTIES fi - -# Run SonarQube scan for Python code -time_command $SONAR_SCANNER - -# Copy the Python scan report-task.txt file -mkdir -p /met/.scannerwork -cp .scannerwork/report-task.txt /met/.scannerwork/python-report-task.txt - -# Copy sonar-project.properties for C/C++ code -[ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES -sed -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \ - -e "s|SONAR_HOST_URL|$SONAR_HOST_URL|" \ - -e "s|SONAR_PROJECT_KEY|MET-GHA-CXX|" \ - -e "s|SONAR_PROJECT_NAME|MET GHA CXX|" \ - -e "s|SONAR_BRANCH_NAME|$MET_GIT_NAME|" \ - -e "s|SONAR_REFERENCE_BRANCH|$SONAR_REFERENCE_BRANCH|" \ - $SONAR_PROPERTIES_DIR/sonar-project.properties > $SONAR_PROPERTIES -# The source and reference branches must differ to define new code -if [ "$MET_GIT_NAME" != "$SONAR_REFERENCE_BRANCH" ]; then - echo "sonar.newCode.referenceBranch=${SONAR_REFERENCE_BRANCH}" >> $SONAR_PROPERTIES -fi - -# Run the configure script +# Run the MET configure script time_command ./configure \ BUFRLIB_NAME=${BUFRLIB_NAME} \ GRIB2CLIB_NAME=${GRIB2CLIB_NAME} \ - ${MET_CONFIG_OPTS} \ + --enable-all \ CPPFLAGS="-I/usr/local/include -I/usr/local/include/freetype2 -I/usr/local/include/cairo" \ LIBS="-ltirpc" @@ -136,11 +111,18 @@ time_command make clean # Run SonarQube make time_command $SONAR_WRAPPER --out-dir $SONARQUBE_OUT_DIR make -# Run SonarQube scan for C/C++ code +# Run SonarQube scan time_command $SONAR_SCANNER +status=$? + +# Check return status +if [[ $status -ne 0 ]]; then + echo "ERROR: ${0} -> the SonarQube scan returned with non-zero status (${status})!" + exit ${status} +fi -# Copy the C/C++ scan report-task.txt file +# Copy the scan report-task.txt file mkdir -p /met/.scannerwork -cp .scannerwork/report-task.txt /met/.scannerwork/cxx-report-task.txt +cp .scannerwork/report-task.txt /met/.scannerwork/report-task.txt [ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES diff --git a/internal/scripts/sonarqube/python.sonar-project.properties b/internal/scripts/sonarqube/python.sonar-project.properties deleted file mode 100644 index 71b6b36b71..0000000000 --- a/internal/scripts/sonarqube/python.sonar-project.properties +++ /dev/null @@ -1,17 +0,0 @@ -sonar.projectKey=SONAR_PROJECT_KEY -sonar.projectName=SONAR_PROJECT_NAME -sonar.projectVersion=1.0 - -sonar.sources=scripts/python,data/wrappers -sonar.python.version=3.6.3 - -# The build-wrapper output dir - -# Encoding of the source files -sonar.sourceEncoding=UTF-8 - -#----- Default SonarQube server -sonar.host.url=SONAR_HOST_URL - -sonar.token=SONAR_TOKEN -sonar.branch.name=SONAR_BRANCH_NAME diff --git a/internal/scripts/sonarqube/run_sonarqube.sh b/internal/scripts/sonarqube/run_sonarqube.sh index 917158b461..1a57b0b726 100755 --- a/internal/scripts/sonarqube/run_sonarqube.sh +++ b/internal/scripts/sonarqube/run_sonarqube.sh @@ -24,10 +24,10 @@ GIT_REPO="https://github.com/dtcenter/${GIT_REPO_NAME}" function usage { - echo - echo "USAGE: $(basename $0) name" - echo " where \"name\" specifies a branch, tag, or hash." - echo + echo + echo "USAGE: $(basename $0) name" + echo " where \"name\" specifies a branch, tag, or hash." + echo } # Check for arguments @@ -90,7 +90,6 @@ function run_command() { return ${STATUS} } - # Store the full path to the scripts directory SCRIPT_DIR=`dirname $0` if [[ ${0:0:1} != "/" ]]; then SCRIPT_DIR=$(pwd)/${SCRIPT_DIR}; fi @@ -113,48 +112,35 @@ export MET_DEVELOPMENT=true # Run the configure script run_command "./configure --prefix=`pwd` --enable-all" -# Set the build id -#BUILD_ID="MET-${1}" +# Define the version string +SONAR_PROJECT_VERSION=$(grep "^version" docs/conf.py | cut -d'=' -f2 | tr -d "\'\" ") SONAR_PROPERTIES=sonar-project.properties -# Copy sonar-project.properties for Python code +# Configure sonar-project.properties [ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES [ -z "$SONAR_HOST_URL" ] && SONAR_HOST_URL="http://localhost:9000" if [ -z "$SONAR_TOKEN" ]; then echo " == ERROR == SONAR_TOKEN is not defined" exit 1 else - sed -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \ - -e "s|SONAR_HOST_URL|$SONAR_HOST_URL|" \ - -e "s|SONAR_PROJECT_KEY|MET_python_NB|" \ - -e "s|SONAR_PROJECT_NAME|MET python Nightly Build|" \ - -e "s|SONAR_BRANCH_NAME|develop|" \ - $SCRIPT_DIR/python.sonar-project.properties > $SONAR_PROPERTIES - - # Run SonarQube scan for Python code - run_command "$SONAR_SCANNER" - - # Copy sonar-project.properties for C/C++ code [ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES - sed -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \ - -e "s|SONAR_HOST_URL|$SONAR_HOST_URL|" \ - -e "s|SONAR_PROJECT_KEY|MET_develop_NB|" \ + sed -e "s|SONAR_PROJECT_KEY|MET_NB|" \ -e "s|SONAR_PROJECT_NAME|MET Nightly Build|" \ - -e "s|SONAR_BRANCH_NAME|develop|" \ - $SCRIPT_DIR/sonar-project.properties > $SONAR_PROPERTIES + -e "s|SONAR_PROJECT_VERSION|$SONAR_PROJECT_VERSION|" \ + -e "s|SONAR_HOST_URL|$SONAR_HOST_URL|" \ + -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \ + -e "s|SONAR_BRANCH_NAME|${1}|" \ + $SCRIPT_DIR/$SONAR_PROPERTIES > $SONAR_PROPERTIES # Run SonarQube clean run_command "make clean" - # Run SonarQube make + # Run SonarQube build wrapper run_command "$SONAR_WRAPPER --out-dir $SONARQUBE_OUT_DIR make" - # Run SonarQube scan for C/C++ code + # Run SonarQube scan run_command "$SONAR_SCANNER" [ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES fi - -# Run SonarQube report generator to make a PDF file -#TODAY=`date +%Y%m%d` diff --git a/internal/scripts/sonarqube/sonar-project.properties b/internal/scripts/sonarqube/sonar-project.properties index c09f1d31b1..215749e75b 100644 --- a/internal/scripts/sonarqube/sonar-project.properties +++ b/internal/scripts/sonarqube/sonar-project.properties @@ -1,17 +1,15 @@ +# Project and source code settings sonar.projectKey=SONAR_PROJECT_KEY sonar.projectName=SONAR_PROJECT_NAME -sonar.projectVersion=1.0 - -sonar.sources=src +sonar.projectVersion=SONAR_PROJECT_VERSION +sonar.branch.name=SONAR_BRANCH_NAME +sonar.sources=src,scripts/python,data/wrappers +sonar.python.version=3.6.3 +sonar.sourceEncoding=UTF-8 # The build-wrapper output dir sonar.cfamily.build-wrapper-output=bw-outputs -# Encoding of the source files -sonar.sourceEncoding=UTF-8 - -#----- Default SonarQube server +# SonarQube server sonar.host.url=SONAR_HOST_URL - sonar.token=SONAR_TOKEN -sonar.branch.name=SONAR_BRANCH_NAME diff --git a/data/colortables/NCL_colortables/rgb2ctable.py b/scripts/python/utility/rgb2ctable.py similarity index 100% rename from data/colortables/NCL_colortables/rgb2ctable.py rename to scripts/python/utility/rgb2ctable.py diff --git a/src/libcode/vx_python3_utils/point.py b/src/libcode/vx_python3_utils/point.py deleted file mode 100644 index 699aa46e51..0000000000 --- a/src/libcode/vx_python3_utils/point.py +++ /dev/null @@ -1,39 +0,0 @@ - - -############################################################### - - -import sys - - -############################################################### - -print ('============= start point.py ===========') - -print (sys.argv) - - -############################################################### - - -point_data = [[ 'ADPUPA', '72365', '20070331_120000', 35.03, -106.62, 1618.0, 7, 837.0, 1618, 'NA', 1618 ], - [ 'ADPUPA', '72365', '20070331_120000', 35.03, -106.62, 1618.0, 11, 837.0, 1618, 'NA', 273.05 ], - [ 'ADPUPA', '72365', '20070331_120000', 35.03, -106.62, 1618.0, 17, 837.0, 1618, 'NA', 271.85 ], - [ 'ADPUPA', '72365', '20070331_120000', 35.03, -106.62, 1618.0, 52, 837.0, 1618, 'NA', 92 ], - [ 'ADPUPA', '72365', '20070331_120000', 35.03, -106.62, 1618.0, 53, 837.0, 1618, 'NA', 0.00417 ], - [ 'ADPUPA', '72365', '20070331_120000', 35.03, -106.62, 1618.0, 7, 826.0, 1724, 'NA', 1724 ], - [ 'ADPUPA', '72365', '20070331_120000', 35.03, -106.62, 1618.0, 11, 826.0, 1724, 'NA', 274.55 ]] - - -############################################################### - - -print (point_data) - - -print ('============= end point.py ===========') - - -############################################################### - -