azure-pipelines.yml

# Variable 'keys.ipstack_auth' was defined in the Variables tab
# Variable 'keys.maxmind_auth' was defined in the Variables tab
# Variable 'PIP_CACHE_DIR' was defined in the Variables tab
# Variable 'python.version' was defined in the Variables tab
# Cron Schedules have been converted using UTC Time Zone and may need to be updated for your location
# Multi-job configuration must be converted to matrix strategy: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#multi-job-configuration
trigger:
  branches:
    include:
    - main
  batch: True
pr:
  branches:
    include:
    - main
    - release/*
schedules:
- cron: 0 4 * * 1
  branches:
    include:
    - main
  always: true
name: $(date:yyyyMMdd)$(rev:.r)
resources:
  repositories:
  - repository: self
    type: git
    ref: main
variables:
  - name: prSource
    ${{ if variables['System.PullRequest.IsFork'] }}:
      value: fork
    ${{ if not(variables['System.PullRequest.IsFork']) }}:
      value: internal
stages:
- stage: PythonTests
  displayName: Python build and tests
  variables:
    # This is run explicity so does not need to be auto-injected
    skipComponentGovernanceDetection: true
  jobs:
  - job: InstallAndTestPackage
    strategy:
      matrix:
        mac:
          imageName: "macos-latest"
          python.version: '3.8'
        windows:
          imageName: "windows-latest"
          python.version: '3.8'
      maxParallel: 4
    pool:
      vmImage: $(imageName)
    steps:
    # Add an alias for Windows python=>python3
    - script: alias python='python3' pip='pip3'
      condition: in(variables['imageName'], 'ubuntu-latest', 'macos-latest')
    - task: UsePythonVersion@0
      displayName: 'Use Python $(python.version)'
      inputs:
        versionSpec: '$(python.version)'
    - task: Cache@2
      displayName: Cache pip packages
      continueOnError: True
      inputs:
        key: python | "$(Agent.OS)"
        path: $(PIP_CACHE_DIR)
        restoreKeys: python
    - script: |
        python -m pip install --upgrade pip wheel setuptools==56.0.0
        python -m pip install -r requirements-all.txt
        python -m pip install -e .
      displayName: 'Install package and dependencies'
    - script: |
        mkdir ~/.msticpy
        mkdir ~/.msticpy/mordor
        cp $(Build.SourcesDirectory)/tests/testdata/geolite/GeoLite2-City.mmdb ~/.msticpy
        touch ~/.msticpy/GeoLite2-City.mmdb
        cp -r $(Build.SourcesDirectory)/tests/testdata/mordor/* ~/.msticpy/mordor
        touch ~/.msticpy/mordor/mitre_tact_cache.pkl
        touch ~/.msticpy/mordor/mitre_tech_cache.pkl
        touch ~/.msticpy/mordor/mordor_cache.pkl
      condition: ne(variables.imageName, 'windows-latest')
      displayName: 'Prepare test dummy data - Linux'
    - script: |
        mkdir %USERPROFILE%\.msticpy
        mkdir %USERPROFILE%\.msticpy\mordor
        copy /Y $(Build.SourcesDirectory)\tests\testdata\geolite\GeoLite2-City.mmdb %USERPROFILE%\.msticpy
        copy /Y $(Build.SourcesDirectory)\tests\testdata\mordor\* %USERPROFILE%\.msticpy\mordor
      condition: eq(variables.imageName, 'windows-latest')
      displayName: 'Prepare test dummy data - Windows'
    - script: |
        echo MSTICPYCONFIG: $(Build.SourcesDirectory)/tests/msticpyconfig-test.yaml
        echo Env $MSTICPYCONFIG or %MSTICPYCONFIG%
        echo Build source: $(prSource)
        echo Env $MSTICPY_BUILD_SOURCE or %MSTICPY_BUILD_SOURCE%
        python -m pip install -r requirements-dev.txt
        python -m pip install "pandas>=1.3.0"
        python -m pip install seaborn
        pytest tests --junitxml=junit/test-$(variables.imageName)-$(variables.python.version)-results.xml -n auto --cov=msticpy --cov-report=xml
      displayName: pytest
      env:
        MSTICPYCONFIG: $(Build.SourcesDirectory)/tests/msticpyconfig-test.yaml
        MAXMIND_AUTH: $(keys.maxmind_auth)
        IPSTACK_AUTH: $(keys.ipstack_auth)
        MSTICPY_TEST_NOSKIP: 1
        MSTICPY_BUILD_SOURCE: $(prSource)
      condition: succeededOrFailed()
      continueOnError: true
    - task: PublishTestResults@2
      displayName: 'Publish Test Results'
      inputs:
        testResultsFiles: '**/*-results.xml'
        testRunTitle: 'Python $(python.version)'
    - task: PublishCodeCoverageResults@1
      displayName: 'Publish code coverage'
      inputs:
        codeCoverageTool: Cobertura
        summaryFileLocation: '$(System.DefaultWorkingDirectory)/coverage.xml'
        reportDirectory: '$(System.DefaultWorkingDirectory)/htmlcov'
  - job: SphinxDocs
    pool:
      vmImage: "ubuntu-latest"
      python.version: '3.8'
    steps:
    - script: |
        python -m pip install sphinx readthedocs-sphinx-ext>=2.1.4 sphinx-rtd-theme>=1.0.0
        python -m pip install -r $(Build.SourcesDirectory)/docs/requirements.txt
        make html
      env:
        SPHINX_NOGEN: "true"
      workingDirectory: docs
      displayName: 'Sphinx Read the Docs build'
- stage: SecurityChecks
  # Pipeline
  displayName: Azure pipeline, security and component governance tools
  variables:
    # This is run explicitly, so does not need to be auto-injected
    skipComponentGovernanceDetection: true
    python.version: '3.8'
  jobs:
  - job: SecurityTests
    # Credscan must be run on Windows
    pool:
      vmImage: windows-latest
    steps:
    - task: CredScan@2
      displayName: 'Run CredScan'
      inputs:
        toolMajorVersion: V2
        debugMode: false
        suppressionsFile: $(Build.SourcesDirectory)\.ci_config\credscan.json
      continueOnError: true
      condition: succeededOrFailed()
    - task: AutoApplicability@1
      inputs:
        ExternalRelease: true
      displayName: 'Run AutoApplicability'
    - task: PoliCheck@1
      displayName: 'Run PoliCheck'
      inputs:
        SOMEnabled: true
        optionsUEPATH: $(Build.SourcesDirectory)\.ci_config\UserExclusion.xml
    - task: VulnerabilityAssessment@0
      displayName: 'Run Vulnerability Assessment'
    - task: ComponentGovernanceComponentDetection@0
      inputs:
        scanType: 'Register'
        verbosity: 'Verbose'
        alertWarningLevel: 'High'
      displayName: 'Component Detection'
    - task: securedevelopmentteam.vss-secure-development-tools.build-task-report.SdtReport@1
      displayName: 'Create Security Analysis Report'
      inputs:
        CredScan: true
        PoliCheck: true
    - task: notice@0
      displayName: NOTICE File Generator
      # This fails for external forks
      condition: not(variables['System.PullRequest.IsFork'])
    - task: PostAnalysis@1
      displayName: Post Analysis
      inputs:
        CredScan: true
        PoliCheck: true
    - task: PublishTestResults@2
      displayName: 'Publish Test Results'
      inputs:
        testResultsFiles: '**/*-results.xml'
        testRunTitle: 'Python $(python.version)-sec'
  - job: AntiMalware
    pool:
      vmImage: "windows-latest"
    steps:
    - task: AntiMalware@3
      inputs:
        InputType: 'Basic'
        ScanType: 'CustomScan'
        FileDirPath: '$(Build.StagingDirectory)'
        EnableServices: true
        SupportLogOnError: false
        TreatSignatureUpdateFailureAs: 'Warning'
        SignatureFreshness: 'UpToDate'
        TreatStaleSignatureAs: 'Error'
      continueOnError: true
      condition: succeededOrFailed()
    - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@2
      displayName: 'Publish Security Analysis Logs'
      inputs:
        AllTools: false
        AntiMalware: true
        APIScan: false
        BinSkim: false
        CodesignValidation: false
        FortifySCA: false
        FxCop: false
        ModernCop: false
        MSRD: false
        RoslynAnalyzers: false
        SDLNativeRules: false
        Semmle: false
        TSLint: false
        WebScout: false
    - task: PublishTestResults@2
      displayName: 'Publish Test Results'
      inputs:
        testResultsFiles: '**/*-results.xml'
        testRunTitle: 'Python $(python.version)-antimalware'
  - job: OWASPCheck
    pool:
      vmImage: "windows-latest"
    steps:
    - task: dependency-check-build-task@5
      displayName: 'OWASP Dependency Check'
      inputs:
        projectName: msticpy
        scanPath: '$(Build.SourcesDirectory)'
        format: HTML,JUNIT
        suppressionPath: '$(Build.SourcesDirectory)'
      continueOnError: true
      condition: succeededOrFailed()
    - task: PublishTestResults@2
      displayName: 'Publish Test Results'
      inputs:
        testResultsFiles: '**/*-results.xml'
        testRunTitle: 'Python $(python.version)-owasp'
- stage: CreatePythonPackage
  displayName: Create msticpy distribution package and publish msticpy artifact
  variables:
    # This is run explicitly so does not need to be auto-injected
    skipComponentGovernanceDetection: true
  dependsOn:
  - SecurityChecks
  condition: succeeded('SecurityChecks')
  jobs:
  - job: CreateDistPackage
    pool:
      vmImage: windows-latest
    variables:
      python.version: '$(python.version)'
    steps:
    - script: 'pip install --upgrade setuptools==56.0.0 wheel'
      displayName: 'Install setuptools'
    - script: 'python setup.py sdist bdist_wheel'
      displayName: 'Build sdist and wheel'
    - task: PublishBuildArtifacts@1
      displayName: 'Publish Artifact: msticpy'
      inputs:
        PathtoPublish: dist
        ArtifactName: msticpy