Add platform-specific test support (and fix testCMCGait10dof18musc.cpp)

Applications/CMC/test/CMakeLists.txt

@@ -1,10 +1,11 @@
+find_package(Catch2 REQUIRED
+        HINTS "${OPENSIM_DEPENDENCIES_DIR}/catch2")
 
 file(GLOB TEST_PROGS "test*.cpp")
 file(GLOB TEST_FILES *.osim *.xml *.sto *.mot)
 
 OpenSimAddTests(
     TESTPROGRAMS ${TEST_PROGS}
     DATAFILES ${TEST_FILES}
-    LINKLIBS osimTools
+    LINKLIBS osimTools Catch2::Catch2WithMain
     )
-

Applications/CMC/test/testCMCGait10dof18musc.cpp

@@ -1,13 +1,13 @@
 /* -------------------------------------------------------------------------- *
- *                           OpenSim:  testCMCGait10dof18musc.cpp             *
+ *                  OpenSim:  testCMCGait10dof18musc.cpp                      *
  * -------------------------------------------------------------------------- *
  * The OpenSim API is a toolkit for musculoskeletal modeling and simulation.  *
  * See http://opensim.stanford.edu and the NOTICE file for more information.  *
  * OpenSim is developed at Stanford University and supported by the US        *
  * National Institutes of Health (U54 GM072970, R24 HD065690) and by DARPA    *
  * through the Warrior Web program.                                           *
  *                                                                            *
- * Copyright (c) 2005-2017 Stanford University and the Authors                *
+ * Copyright (c) 2005-2023 Stanford University and the Authors                *
  *                                                                            *
  * Licensed under the Apache License, Version 2.0 (the "License"); you may    *
  * not use this file except in compliance with the License. You may obtain a  *
@@ -21,67 +21,69 @@
  * -------------------------------------------------------------------------- */
 
 // INCLUDE
+#include <OpenSim/Common/GCVSplineSet.h>
 #include <OpenSim/Simulation/Model/Model.h>
-#include <OpenSim/Simulation/Model/AnalysisSet.h>
 #include <OpenSim/Tools/CMCTool.h>
-#include <OpenSim/Tools/ForwardTool.h>
-#include <OpenSim/Auxiliary/auxiliaryTestFunctions.h>
 
-using namespace OpenSim;
-using namespace std;
-
-void testGait10dof18musc();
-
-int main() {
+#include <catch2/catch_all.hpp>
 
-    SimTK::Array_<std::string> failures;
-
-    // Model uses Millard2012EquilibriumMuscle type muscles
-    try { testGait10dof18musc(); }
-    catch (const std::exception& e) {
-        cout << e.what() << endl;
-        failures.push_back("testGait10dof18musc");
-    }
+using namespace OpenSim;
 
-    if (!failures.empty()) {
-        cout << "Done, with failure(s): " << failures << endl;
-        return 1;
+TEST_CASE("testGait10dof18musc (Windows)", "[win]") {
+    CMCTool cmc("gait10dof18musc_Setup_CMC.xml");
+    cmc.run();
+
+    const TimeSeriesTable results(
+        "gait10dof18musc_ResultsCMC/walk_subject_states.sto");
+    const TimeSeriesTable std(
+        "gait10dof18musc_std_walk_subject_states_win.sto");
+
+    // TODO: Replace with macro from OpenSim/Moco/Test/Testing.h
+    const auto& actual = results.getMatrix();
+    const auto& expected = std.getMatrix();
+    REQUIRE((actual.nrow() == expected.nrow()));
+    REQUIRE((actual.ncol() == expected.ncol()));
+    for (int ir = 0; ir < actual.nrow(); ++ir) {
+        for (int ic = 0; ic < actual.ncol(); ++ic) {
+            INFO("(" << ir << "," << ic << "): " << actual.getElt(ir, ic) <<
+                " vs " << expected.getElt(ir, ic));
+            REQUIRE((Catch::Approx(actual.getElt(ir, ic)).margin(1e-3)
+                == expected.getElt(ir, ic)));
+        }
     }
-
-    cout << "Done" << endl;
-
-    return 0;
 }
 
-// Perform regression test with standard generated from Gait10dof18musc
-// example in OpenSim 3.2
-void testGait10dof18musc() {
-    cout<<"\n******************************************************************" << endl;
-    cout << "*                      testGait10dof18musc                       *" << endl;
-    cout << "******************************************************************\n" << endl;
+TEST_CASE("testGait10dof18musc (Mac/Linux)", "[unix]") {
     CMCTool cmc("gait10dof18musc_Setup_CMC.xml");
-    const string& muscleType = cmc.getModel().getMuscles()[0].getConcreteClassName();
-
-    if (!cmc.run())
-        OPENSIM_THROW(Exception, "testGait10dof18musc " + muscleType +
-            " failed to complete.");
-
-    Storage results("gait10dof18musc_ResultsCMC/walk_subject_states.sto");
-    Storage temp("gait10dof18musc_std_walk_subject_states.sto");
-
-    Storage *standard = new Storage();
-    cmc.getModel().formStateStorage(temp, *standard);
-
-    int nstates = standard->getColumnLabels().size() - 1;
-
-    // angles and speeds within 0.01 rads and 0.01 rad/s;
-    // and activations to within 1%
-    std::vector<double> rms_tols(nstates, 0.01);
-
-    CHECK_STORAGE_AGAINST_STANDARD(results, *standard, rms_tols,
-        __FILE__, __LINE__, "testGait10dof18musc "+ muscleType + " failed");
-
-    cout << "\ntestGait10dof18musc "+ muscleType +" passed\n" << endl;
+    cmc.run();
+
+    const TimeSeriesTable results(
+        "gait10dof18musc_ResultsCMC/walk_subject_states.sto");
+    const TimeSeriesTable std(
+        "gait10dof18musc_std_walk_subject_states_unix.sto");
+
+    // Unix systems produce inconsistent results compared to Windows. Somehow,
+    // CMC produces results with differing number of time points, so we need to
+    // interpolate the results to the same time points as the standard results.
+    // The shapes of the muscle activity curves also differ slightly, so we
+    // allow a larger margin of error for the muscle activity curves. Therefore,
+    // this is a weaker test compared to the Windows test.
+    GCVSplineSet resultSplines(results);
+    GCVSplineSet stdSplines(std);
+
+    // TODO: Replace with (a new) macro from OpenSim/Moco/Test/Testing.h
+    const auto& time = std.getIndependentColumn();
+    for (const auto& label : results.getColumnLabels()) {
+        const auto& result = resultSplines.get(label);
+        const auto& expected = stdSplines.get(label);
+        SimTK::Vector timeVec(1, 0.0);
+        for (int it = 0; it < static_cast<int>(time.size()); ++it) {
+            timeVec[0] = time[it];
+            INFO(label << " at time " << time[it] << ": " <<
+                result.calcValue(timeVec) << " vs " <<
+                expected.calcValue(timeVec));
+            REQUIRE((Catch::Approx(result.calcValue(timeVec)).margin(1e-1)
+                == expected.calcValue(timeVec)));
+        }
+    }
 }
-
-

CONTRIBUTING.md

@@ -83,21 +83,59 @@ that are already in `OpenSim/Tests/shared`; you could inadvertently weaken tests
 that rely on some obscure aspect of the files.
 
 
+Platform-specific tests
+-----------------------
+The Catch2 testing framework can be used to write platform-specific tests. Such
+tests should be avoided if possible, but may be necessary in certain cases (e.g.,
+slightly different numerical optimization behavior on different platforms). For
+example, if you want to write a test that only runs on Windows, you can use the
+`TEST_CASE()` or `TEMPLATE_TEST_CASE` macros with the tag `"[win]"`:
+
+```cpp
+TEST_CASE("MyTest", "[win]") {
+    // ...
+}
+```
+
+```cpp
+TEMPLATE_TEST_CASE("MyTest", "[win]", TestType) {
+    // ...
+}
+```
+
+Specifying the tag `"[win]"` means that this test will be _excluded_ on Mac and
+Linux. The tags `"[mac]"` and `"[linux]"` can be used similarly for tests specific 
+to Mac or Linux. If you want to run a test on two platforms but not the third,
+use combined tags (e.g., `"[win/linux]"`, `"[mac/win]"`, or `"[unix]"`). Do not
+concatenate tags; for example, `"[win][linux]"`) will not run on Linux or Windows
+since Windows excludes Linux-only tests and vice-versa. Specifying no tag means 
+that the test will run on all platforms. The table below summarizes the tags that 
+can be used to specify platform-specific tests.
+
+| Platform(s)       | Tag(s)                                       |
+|-------------------|----------------------------------------------|
+| Windows           | `"[win]"`                                    |
+| Mac               | `"[mac]"`                                    |
+| Linux             | `"[linux]"`                                  |
+| Mac and Linux     | `"[unix]"`, `"[mac/linux]"`, `"[linux/mac]"` |
+| Windows and Mac   | `"[win/mac]"`, `"[mac/win]"`                 |
+| Windows and Linux | `"[win/linux]"`, `"[linux/win]"`             |
+| All platforms     | (no tag)                                     |
+
+
 Running Moco tests
 ------------------
 In general, Moco's tests depend on the CasADi and Tropter libraries, whose use
 is determined by the `OPENSIM_WITH_CASADI` and `OPENSIM_WITH_TROPTER` CMake
 variables. The CTests are designed to succeed regardless of the value of these
 CMake variables: if `OPENSIM_WITH_CASADI` is off, Moco's C++ tests are run with
-arguments `"exclude:*MocoCasADiSolver*" "exclude:[casadi]"`,
-which excludes Catch2 templatized tests using MocoCasADiSolver and other tests
-that are tagged as relying on CasADi (likewise for Tropter).
-If the test executables are run without CTest (e.g., debugging a project in
-Visual Studio), the tests will fail if either `OPENSIM_WITH_CASADI` or
-`OPENSIM_WITH_TROPTER` is false; for the tests to pass, provide the argument(s)
-`"exclude:*MocoCasADiSolver*" "exclude:[casadi]"` and/or
-`"exclude:*MocoTropterSolver*" "exclude:[tropter]"` (depending on which
-libraries are available).
+arguments `"~*MocoCasADiSolver*" "~[casadi]"`, which excludes Catch2 templatized 
+tests using MocoCasADiSolver and other tests that are tagged as relying on CasADi 
+(likewise for Tropter). If the test executables are run without CTest (e.g., 
+debugging a project in Visual Studio), the tests will fail if either 
+`OPENSIM_WITH_CASADI` orv`OPENSIM_WITH_TROPTER` is false; for the tests to pass, 
+provide the argument(s) `"~*MocoCasADiSolver*" "~[casadi]"` and/or
+`"~*MocoTropterSolver*" "~[tropter]"` (depending on which libraries are available).
 
 
 Checking for Memory Leaks with LibASAN

OpenSim/Common/Test/CMakeLists.txt

@@ -23,5 +23,5 @@ OpenSimAddTests(
     TESTPROGRAMS ${TEST_PROGS}
     DATAFILES ${TEST_FILES} ${C3D_TEST_FILES} ${TRC_TEST_FILES} ${XSENS_TEST_FILES} ${APDM_TEST_FILES}
               ${MOT_TEST_FILES}
-    LINKLIBS osimCommon ${SIMTK_ALL_LIBS} Catch2::Catch2WithMain
+    LINKLIBS osimCommon Catch2::Catch2WithMain ${SIMTK_ALL_LIBS}
     )

OpenSim/Moco/Test/CMakeLists.txt

@@ -17,12 +17,12 @@ function(MocoAddTest)
     target_link_libraries(${MOCOTEST_NAME} osimMoco ${MOCOTEST_LIB_DEPENDS} Catch2::Catch2WithMain)
     # Skip tests relying on MocoCasADiSolver if CasADi is not available.
     if(NOT OPENSIM_WITH_CASADI)
-        list(APPEND test_args "exclude:*MocoCasADiSolver*")
-        list(APPEND test_args "exclude:[casadi]")
+        list(APPEND test_args "~*MocoCasADiSolver*")
+        list(APPEND test_args "~[casadi]")
     endif()
     if(NOT OPENSIM_WITH_TROPTER)
-        list(APPEND test_args "exclude:*MocoTropterSolver*")
-        list(APPEND test_args "exclude:[tropter]")
+        list(APPEND test_args "~*MocoTropterSolver*")
+        list(APPEND test_args "~[tropter]")
     endif()
     add_test(NAME ${MOCOTEST_NAME} COMMAND ${MOCOTEST_NAME} ${test_args})
     # Tests likely run faster if we parallelize the tests rather than the
@@ -62,8 +62,6 @@ MocoAddTest(NAME testMocoTrack RESOURCES
         std_testMocoTrackGait10dof18musc_solution.sto
         )
 
-# MocoAddTest(NAME testMultivariatePolynomial)
-
 MocoAddTest(NAME testMocoAnalytic)
 
 MocoAddTest(NAME testMocoMetabolics)

cmake/OpenSimMacros.cmake

@@ -223,9 +223,9 @@ function(OpenSimAddLibrary)
 
     # This is for exporting classes on Windows.
     if(OSIMADDLIB_VENDORLIB)
-	    set(OSIMADDLIB_FOLDER "Vendor Libraries")
+        set(OSIMADDLIB_FOLDER "Vendor Libraries")
     else()
-		set(OSIMADDLIB_FOLDER "Libraries")
+        set(OSIMADDLIB_FOLDER "Libraries")
     endif()
     set_target_properties(${OSIMADDLIB_LIBRARY_NAME} PROPERTIES
        DEFINE_SYMBOL OSIM${OSIMADDLIB_UKIT}_EXPORTS
@@ -334,7 +334,7 @@ endfunction()
 #   file(GLOB TEST_PROGRAMS "test*.cpp")
 #   file(GLOB DATA_FILES *.osim *.xml *.sto *.mot)
 #   OpenSimAddTests(
-#       TESTPROGRAMS ${TEST_ROGRAMS}
+#       TESTPROGRAMS ${TEST_PROGRAMS}
 #       DATAFILES ${DATA_FILES}
 #       LINKLIBS osimCommon osimSimulation osimAnalyses
 #       )
@@ -367,11 +367,20 @@ function(OpenSimAddTests)
             add_executable(${TEST_NAME} ${test_program}
                 ${OSIMADDTESTS_SOURCES})
             target_link_libraries(${TEST_NAME} ${OSIMADDTESTS_LINKLIBS})
-            add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME})
+            set(test_args "")
+            if(APPLE)
+                list(APPEND test_args "~[win]~[linux]~[win/linux]~[linux/win]")
+            endif()
+            if(LINUX)
+                list(APPEND test_args "~[win]~[mac]~[win/mac]~[mac/win]")
+            endif()
+            if(WIN32)
+                list(APPEND test_args "~[mac]~[linux]~[mac/linux]~[linux/mac]~[unix]")
+            endif()
+            add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME} ${test_args})
             set_target_properties(${TEST_NAME} PROPERTIES
                 FOLDER "Tests"
-				)
-
+            )
         endforeach()
 
         # Copy data files to build directory.