Regression tests

.github/workflows/regressionTests.yml

@@ -0,0 +1,61 @@
+name: Regression Tests
+
+on:
+  push:
+    branches: ['master', 'maintenance/*']
+  pull_request:
+  schedule:
+    - cron: "25 4 * * 3" # Every Wednesday at 04:25
+  workflow_dispatch:
+
+jobs:
+  regression-test:
+    if: github.repository == 'OpenModelica/OMJulia.jl' # Run only on OpenModelica/OMJulia.jl to prevent spamming forks
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        julia-version: ['1.8', '1.9']
+        julia-arch: ['x64']
+        os: ['ubuntu-latest', 'windows-latest']
+        omc-version: ['stable', 'nightly', '1.21']
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: "Set up OpenModelica Compiler"
+        uses: AnHeuermann/[email protected]
+        with:
+          version: ${{ matrix.omc-version }}
+          packages: |
+            omc
+          libraries: |
+            'Modelica 4.0.0'
+
+      - run: "omc --version"
+
+      - name: "Set up Julia"
+        uses: julia-actions/setup-julia@v1
+        with:
+          version: ${{ matrix.julia-version }}
+          arch: ${{ matrix.julia-arch }}
+
+      - name: Cache Julia
+        uses: julia-actions/cache@v1
+
+      - name: "Build OMJulia"
+        uses: julia-actions/julia-buildpkg@v1
+
+      - name: Install dependencies
+        run: julia --project=regression-tests/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
+
+      - name: "Run regression test"
+        shell: bash
+        run: julia --project=regression-tests/. -e 'include("regression-tests/regressionTests.jl"); runTests(libraries, models)'
+
+      - name: Archive FMUs
+        uses: actions/upload-artifact@v3
+        with:
+          name: fmu-export
+          path: regression-tests/temp/**/*.fmu

README.md

@@ -2,7 +2,7 @@
 
 *Julia scripting [OpenModelica](https://openmodelica.org/) interface.*
 
-[![][docs-dev-img]][docs-dev-url] [![][GHA-test-img]][GHA-test-url]
+[![][docs-dev-img]][docs-dev-url] [![][GHA-test-img]][GHA-test-url] [![][GHA-regressions-img]][GHA-regressions-url]
 
 ## Requirements
 
@@ -82,3 +82,6 @@ CONDITIONS OF OSMC-PL.
 
 [GHA-test-img]: https://github.com/OpenModelica/OMJulia.jl/actions/workflows/Test.yml/badge.svg?branch=master
 [GHA-test-url]: https://github.com/OpenModelica/OMJulia.jl/actions/workflows/Test.yml
+
+[GHA-regressions-img]: https://github.com/OpenModelica/OMJulia.jl/actions/workflows/regressionTests.yml/badge.svg?branch=master
+[GHA-regressions-url]: https://github.com/OpenModelica/OMJulia.jl/actions/workflows/regressionTests.yml

regression-tests/.gitignore

@@ -0,0 +1 @@
+temp/

regression-tests/Project.toml

@@ -0,0 +1,6 @@
+[deps]
+CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
+FMI = "14a09403-18e3-468f-ad8a-74f8dda2d9ac"
+OMJulia = "0f4fe800-344e-11e9-2949-fb537ad918e1"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

regression-tests/regressionTests.jl

@@ -0,0 +1,145 @@
+#=
+This file is part of OpenModelica.
+Copyright (c) 1998-2023, Open Source Modelica Consortium (OSMC),
+c/o Linköpings universitet, Department of Computer and Information Science,
+SE-58183 Linköping, Sweden.
+
+All rights reserved.
+
+THIS PROGRAM IS PROVIDED UNDER THE TERMS OF THE BSD NEW LICENSE OR THE
+GPL VERSION 3 LICENSE OR THE OSMC PUBLIC LICENSE (OSMC-PL) VERSION 1.2.
+ANY USE, REPRODUCTION OR DISTRIBUTION OF THIS PROGRAM CONSTITUTES
+RECIPIENT'S ACCEPTANCE OF THE OSMC PUBLIC LICENSE OR THE GPL VERSION 3,
+ACCORDING TO RECIPIENTS CHOICE.
+
+The OpenModelica software and the OSMC (Open Source Modelica Consortium)
+Public License (OSMC-PL) are obtained from OSMC, either from the above
+address, from the URLs: http://www.openmodelica.org or
+http://www.ida.liu.se/projects/OpenModelica, and in the OpenModelica
+distribution. GNU version 3 is obtained from:
+http://www.gnu.org/copyleft/gpl.html. The New BSD License is obtained from:
+http://www.opensource.org/licenses/BSD-3-Clause.
+
+This program is distributed WITHOUT ANY WARRANTY; without even the implied
+warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, EXCEPT AS
+EXPRESSLY SET FORTH IN THE BY RECIPIENT SELECTED SUBSIDIARY LICENSE
+CONDITIONS OF OSMC-PL.
+=#
+
+using Test
+
+"""
+Run single test process.
+
+Start a new Julia process.
+Kill process and throw an error when timeout is reached.
+Catch InterruptException, kill process and rethorw InterruptException.
+
+# Arguments
+  - `library`:  Modelica library name.
+  - `version`:  Library version.
+  - `model`:    Modelica model from library to test.
+  - `testdir`:  Test working directory.
+
+# Keywords
+  - `timeout=10*60::Integer`:   Timeout in seconds. Defaults to 10 minutes.
+"""
+function singleTest(library, version, model, testdir;
+                    timeout=10*60::Integer)
+
+  mkpath(testdir)
+  logFile = joinpath(testdir, "runSingleTest.log")
+  rm(logFile, force=true)
+
+  @info "Testing $model"
+
+  cmd = Cmd(`$(joinpath(Sys.BINDIR, "julia")) runSingleTest.jl $(library) $(version) $(model) $(testdir)`, dir=@__DIR__)
+  @info cmd
+  plp = pipeline(cmd, stdout=logFile, stderr=logFile)
+  process = run(plp, wait=false)
+
+  try
+    timer = Timer(0; interval=1)
+    for _ in 1:timeout
+      wait(timer)
+      if !process_running(process)
+        close(timer)
+        break
+      end
+    end
+    if process_running(process)
+      @error "Killing $(process)"
+      kill(process)
+    end
+  catch e
+    if isa(e, InterruptException) && process_running(p)
+      @error "Killing process $(cmd)."
+      kill(p)
+    end
+    rethrow(e)
+  end
+
+  println(read(logFile, String))
+
+  status = (process.exitcode == 0) &&
+           isfile(joinpath(testdir, "$(model).fmu")) &&
+           isfile(joinpath(testdir, "FMI_results.csv"))
+
+  return status
+end
+
+"""
+Run all tests.
+
+# Arguments
+  - `libraries::Vector{Tuple{S,S}}`:  Vector of tuples with library and version to test.
+  - `models::Vector{Vector{S}}`:      Vector of vectors with models to test for each library.
+
+# Keywords
+  - `workdir`:                        Root working directory.
+"""
+function runTests(libraries::Vector{Tuple{S,S}},
+                  models::Vector{Vector{S}};
+                  workdir=abspath(joinpath(@__DIR__, "temp"))) where S<:AbstractString
+
+  rm(workdir, recursive=true, force=true) # This can break on Windows when some program or file is still open
+  mkpath(workdir)
+
+  @testset "OpenModelica" begin
+    for (i, (library, version)) in enumerate(libraries)
+      @testset verbose=true "$library" begin
+        libdir = joinpath(workdir, library)
+        mkpath(libdir)
+
+        for model in models[i]
+          modeldir = joinpath(libdir, model)
+          @testset "$model" begin
+            @test singleTest(library, version, model, modeldir)
+          end
+        end
+      end
+    end
+  end
+
+  return
+end
+
+libraries = [
+  ("Modelica", "4.0.0")
+]
+
+models = [
+  [
+    "Modelica.Blocks.Examples.Filter",
+    "Modelica.Electrical.Analog.Examples.CauerLowPassAnalog",
+    "Modelica.Blocks.Examples.RealNetwork1",
+    "Modelica.Electrical.Digital.Examples.FlipFlop",
+    "Modelica.Mechanics.Rotational.Examples.FirstGrounded",
+    "Modelica.Mechanics.Rotational.Examples.CoupledClutches",
+    "Modelica.Mechanics.MultiBody.Examples.Elementary.DoublePendulum",
+    "Modelica.Mechanics.MultiBody.Examples.Elementary.FreeBody",
+    "Modelica.Fluid.Examples.TraceSubstances.RoomCO2WithControls",
+    "Modelica.Clocked.Examples.SimpleControlledDrive.ClockedWithDiscreteTextbookController",
+    "Modelica.Fluid.Examples.PumpingSystem"
+  ]
+]

regression-tests/runSingleTest.jl

@@ -0,0 +1,145 @@
+#=
+This file is part of OpenModelica.
+Copyright (c) 1998-2023, Open Source Modelica Consortium (OSMC),
+c/o Linköpings universitet, Department of Computer and Information Science,
+SE-58183 Linköping, Sweden.
+
+All rights reserved.
+
+THIS PROGRAM IS PROVIDED UNDER THE TERMS OF THE BSD NEW LICENSE OR THE
+GPL VERSION 3 LICENSE OR THE OSMC PUBLIC LICENSE (OSMC-PL) VERSION 1.2.
+ANY USE, REPRODUCTION OR DISTRIBUTION OF THIS PROGRAM CONSTITUTES
+RECIPIENT'S ACCEPTANCE OF THE OSMC PUBLIC LICENSE OR THE GPL VERSION 3,
+ACCORDING TO RECIPIENTS CHOICE.
+
+The OpenModelica software and the OSMC (Open Source Modelica Consortium)
+Public License (OSMC-PL) are obtained from OSMC, either from the above
+address, from the URLs: http://www.openmodelica.org or
+http://www.ida.liu.se/projects/OpenModelica, and in the OpenModelica
+distribution. GNU version 3 is obtained from:
+http://www.gnu.org/copyleft/gpl.html. The New BSD License is obtained from:
+http://www.opensource.org/licenses/BSD-3-Clause.
+
+This program is distributed WITHOUT ANY WARRANTY; without even the implied
+warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, EXCEPT AS
+EXPRESSLY SET FORTH IN THE BY RECIPIENT SELECTED SUBSIDIARY LICENSE
+CONDITIONS OF OSMC-PL.
+=#
+
+import Pkg; Pkg.activate(@__DIR__)
+import OMJulia
+import FMI
+
+using Test
+using DataFrames
+using CSV
+
+"""
+Simulate single model to generate a result file.
+"""
+function testSimulation(omc::OMJulia.OMCSession, className::String)
+  @info "\tSimulation"
+  @testset "Simulation" begin
+    res = OMJulia.API.simulate(omc, className; outputFormat="csv")
+    resultFile = res["resultFile"]
+
+    @test isfile(resultFile)
+    return resultFile
+  end
+end
+
+"""
+Build a FMU for a single model, import the generated FMU, simulate it and compare to given reference results.
+"""
+function testFmuExport(omc::OMJulia.OMCSession, className::String, referenceResult, recordValues; workdir::String)
+  fmuPath = ""
+  fmuImportSuccess = false
+  @info "\tFMU Export"
+  @testset "Export" begin
+    fmuPath = OMJulia.API.buildModelFMU(omc, className)
+    @test isfile(fmuPath)
+    @test splitext(splitpath(fmuPath)[end]) == (className, ".fmu")
+  end
+
+  @info "\tFMU Import"
+  @testset "Import" begin
+    if isfile(fmuPath)
+      fmu = FMI.fmiLoad(fmuPath)
+      solution = FMI.fmiSimulate(fmu; recordValues = recordValues, showProgress=false)
+
+      # Own implementation of CSV export, workaround for https://github.com/ThummeTo/FMI.jl/issues/198
+      df = DataFrames.DataFrame(time = solution.values.t)
+      for i in 1:length(solution.values.saveval[1])
+        for var in FMI.fmi2ValueReferenceToString(fmu, solution.valueReferences[i])
+          if in(var, recordValues)
+            df[!, Symbol(var)] = [val[i] for val in solution.values.saveval]
+          end
+        end
+      end
+      fmiResult = joinpath(workdir, "FMI_results.csv")
+      CSV.write(fmiResult, df)
+
+      #FMI.fmiSaveSolution(solution, "FMI_results.csv")
+      fmuImportSuccess = true
+    end
+    @test fmuImportSuccess
+  end
+
+  @info "\tCheck Results"
+  @testset "Verification" begin
+    if fmuImportSuccess
+      @test (true, String[]) == OMJulia.API.diffSimulationResults(omc, "FMI_results.csv", referenceResult, "diff")
+    else
+      @test false
+    end
+  end
+end
+
+"""
+Run Simulation and FMU export/import test for all models.
+"""
+function runSingleTest(library, version, model, modeldir)
+  local resultFile
+
+  @info "Testing library: $library, model $model"
+  mkpath(modeldir)
+  omc = OMJulia.OMCSession()
+
+  try
+    @testset "$model" verbose=true begin
+      @testset "Simulation" begin
+        OMJulia.API.cd(omc, modeldir)
+
+        @test OMJulia.API.loadModel(omc, library; priorityVersion = [version], requireExactVersion = true)
+        resultFile = testSimulation(omc, model)
+      end
+
+      @testset "FMI" begin
+        if isfile(resultFile)
+          recordValues = names(CSV.read(resultFile, DataFrame))[2:end]
+          filter!(val -> !startswith(val, "\$"), recordValues) # Filter internal variables
+          testFmuExport(omc, model, resultFile, recordValues; workdir=modeldir)
+        else
+          @test false
+        end
+      end
+    end
+  finally
+    OMJulia.quit(omc)
+  end
+end
+
+# Comand-line interface
+if !isempty(PROGRAM_FILE)
+  if length(ARGS) == 4
+    library  = ARGS[1]
+    version  = ARGS[2]
+    model    = ARGS[3]
+    modeldir = ARGS[4]
+    runSingleTest(library, version, model, modeldir)
+  else
+    @error "Wrong number of arguments"
+    for a in ARGS; println(a); end
+    return -1
+  end
+end