singleNode12Rank reproduces convergence an timing on Sunspot.

examples/fluids/navierstokes.c

@@ -294,7 +294,7 @@ int main(int argc, char **argv) {
     app_ctx->cont_steps = stepNumCGNSread;
   }
   PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Before TSSolve navierstokes.c : \n"));
-  PetscCall(TSSolve_NS(dm, user, app_ctx, phys_ctx, &Q, &final_time, &ts));
+  PetscCall(TSSolve_NS(dm, user, app_ctx, phys_ctx, problem, &Q, &final_time, &ts));
 
   // ---------------------------------------------------------------------------
   // Post-processing

examples/fluids/navierstokes.h

@@ -322,7 +322,7 @@ PetscErrorCode UpdateBoundaryValues(User user, Vec Q_loc, PetscReal t);
 PetscErrorCode CreateDM(MPI_Comm comm, ProblemData problem, MatType, VecType, DM *dm);
 
 // Set up DM
-PetscErrorCode SetUpDM(DM *dm, ProblemData *problem, PetscInt degree, PetscInt q_extra, SimpleBC bc, Physics phys);
+PetscErrorCode SetUpDM(DM *dm, ProblemData problem, PetscInt degree, PetscInt q_extra, SimpleBC bc, Physics phys);
 PetscErrorCode DMSetupByOrderBegin_FEM(PetscBool setup_faces, PetscBool setup_coords, PetscInt degree, PetscInt coord_order, PetscInt q_extra,
                                        PetscInt num_fields, const PetscInt *field_sizes, DM dm);
 PetscErrorCode DMSetupByOrderEnd_FEM(PetscBool setup_coords, DM dm);

examples/fluids/problems/stg_shur14.c

@@ -251,7 +251,6 @@ PetscErrorCode SetupStg(const MPI_Comm comm, const DM dm, ProblemData problem, U
   global_stg_ctx->use_fluctuating_IC = use_fluctuating_IC;
   global_stg_ctx->theta0             = theta0;
   global_stg_ctx->P0                 = P0;
-  global_stg_ctx->h_scale_factor     = stg_h_scale_factor;
 
   {  // Calculate dx assuming constant spacing
     PetscReal domain_min[3], domain_max[3], domain_size[3];

examples/fluids/qfunctions/bc_freestream.h

@@ -44,7 +44,7 @@ CEED_QFUNCTION_HELPER int Freestream(void *ctx, CeedInt Q, const CeedScalar *con
         flux = RiemannFlux_HLL(newt_ctx, s, context->S_infty, normal);
         break;
       case RIEMANN_HLLC:
-        flux = RiemannFlux_HLLC(newt_ctx, s, context->S_infty, norm);
+        flux = RiemannFlux_HLLC(newt_ctx, s, context->S_infty, normal);
         break;
     }
     CeedScalar Flux[5];
@@ -114,7 +114,7 @@ CEED_QFUNCTION_HELPER int Freestream_Jacobian(void *ctx, CeedInt Q, const CeedSc
         dflux = RiemannFlux_HLL_fwd(newt_ctx, s, ds, context->S_infty, dS_infty, normal);
         break;
       case RIEMANN_HLLC:
-        dflux = RiemannFlux_HLLC_fwd(newt_ctx, s, ds, context->S_infty, dS_infty, norm);
+        dflux = RiemannFlux_HLLC_fwd(newt_ctx, s, ds, context->S_infty, dS_infty, normal);
         break;
     }
     CeedScalar dFlux[5];

examples/fluids/qfunctions/stg_shur14.h

@@ -12,11 +12,9 @@
 /// SetupSTG_Rand reads in the input files and fills in STGShur14Context.
 /// Then STGShur14_CalcQF is run over quadrature points.
 /// Before the program exits, TearDownSTG is run to free the memory of the allocated arrays.
-#include <ceed/types.h>
-#ifndef CEED_RUNNING_JIT_PASS
+#include <ceed.h>
 #include <math.h>
 #include <stdlib.h>
-#endif
 
 #include "newtonian_state.h"
 #include "setupgeo_helpers.h"
@@ -104,15 +102,12 @@ CEED_QFUNCTION_HELPER CeedScalar Calc_qn(const CeedScalar kappa, const CeedScala
 }
 
 // Calculate hmax, ke, keta, and kcut
-CEED_QFUNCTION_HELPER void SpectrumConstants(const CeedScalar wall_dist, const CeedScalar eps, const CeedScalar lt, const CeedScalar hNodSep[3],
 CEED_QFUNCTION_HELPER void SpectrumConstants(const CeedScalar wall_dist, const CeedScalar eps, const CeedScalar lt, const CeedScalar hNodSep[3],
                                              const CeedScalar nu, CeedScalar *hmax, CeedScalar *ke, CeedScalar *keta, CeedScalar *kcut) {
   *hmax = Max(Max(hNodSep[0], hNodSep[1]), hNodSep[2]);
-  *hmax = Max(Max(hNodSep[0], hNodSep[1]), hNodSep[2]);
   *ke   = wall_dist == 0 ? 1e16 : 2 * M_PI / Min(2 * wall_dist, 3 * lt);
   *keta = 2 * M_PI * pow(Cube(nu) / eps, -0.25);
   *kcut = M_PI / Min(Max(Max(hNodSep[1], hNodSep[2]), 0.3 * (*hmax)) + 0.1 * wall_dist, *hmax);
-  *kcut = M_PI / Min(Max(Max(hNodSep[1], hNodSep[2]), 0.3 * (*hmax)) + 0.1 * wall_dist, *hmax);
 }
 
 /*
@@ -318,11 +313,22 @@ CEED_QFUNCTION(ICsStg)(void *ctx, CeedInt Q, const CeedScalar *const *in, CeedSc
       for (CeedInt j = 0; j < 3; j++) u[j] = ubar[j];
     }
 
-    CeedScalar Y[5] = {P0, u[0], u[1], u[2], theta0}, q[5];
-    State      s    = StateFromY(gas, Y);
-    StateToQ(gas, s, q, gas->state_var);
-    for (CeedInt j = 0; j < 5; j++) {
-      q0[j][i] = q[j];
+    switch (stg_ctx->newtonian_ctx.state_var) {
+      case STATEVAR_CONSERVATIVE:
+        q0[0][i] = rho;
+        q0[1][i] = u[0] * rho;
+        q0[2][i] = u[1] * rho;
+        q0[3][i] = u[2] * rho;
+        q0[4][i] = rho * (0.5 * Dot3(u, u) + cv * theta0);
+        break;
+
+      case STATEVAR_PRIMITIVE:
+        q0[0][i] = P0;
+        q0[1][i] = u[0];
+        q0[2][i] = u[1];
+        q0[3][i] = u[2];
+        q0[4][i] = theta0;
+        break;
     }
   }
   return 0;
@@ -521,22 +527,22 @@ CEED_QFUNCTION(StgShur14InflowStrongQF)(void *ctx, CeedInt Q, const CeedScalar *
       for (CeedInt j = 0; j < 3; j++) u[j] = ubar[j];
     }
 
-    CeedScalar Y[5] = {P0, u[0], u[1], u[2], theta0}, q[5];
-    State      s    = StateFromY(gas, Y);
-    StateToQ(gas, s, q, gas->state_var);
-    switch (gas->state_var) {
+    switch (stg_ctx->newtonian_ctx.state_var) {
       case STATEVAR_CONSERVATIVE:
-        q[4] = 0.;  // Don't set energy
+        bcval[0][i] = scale[i] * rho;
+        bcval[1][i] = scale[i] * rho * u[0];
+        bcval[2][i] = scale[i] * rho * u[1];
+        bcval[3][i] = scale[i] * rho * u[2];
+        bcval[4][i] = 0.;
         break;
+
       case STATEVAR_PRIMITIVE:
-        q[0] = 0;  // Don't set pressure
+        bcval[0][i] = 0;
+        bcval[1][i] = scale[i] * u[0];
+        bcval[2][i] = scale[i] * u[1];
+        bcval[3][i] = scale[i] * u[2];
+        bcval[4][i] = scale[i] * theta0;
         break;
-      case STATEVAR_ENTROPY:
-        q[0] = 0;  // Don't set V_density
-        break;
-    }
-    for (CeedInt j = 0; j < 5; j++) {
-      bcval[j][i] = scale[i] * q[j];
     }
   }
   return 0;

examples/fluids/qfunctions/stg_shur14_type.h

@@ -6,10 +6,7 @@
 // This file is part of CEED:  http://github.com/ceed
 #pragma once
 
-#include <ceed/types.h>
-#ifndef CEED_RUNNING_JIT_PASS
-#include <stdbool.h>
-#endif
+#include <ceed.h>
 
 #include "newtonian_types.h"
 

examples/fluids/src/setupdm.c

@@ -50,7 +50,7 @@ static void evaluate_solution(PetscInt dim, PetscInt Nf, PetscInt NfAux, const P
 }
 
 // Setup DM
-PetscErrorCode SetUpDM(DM *dm, ProblemData *problem, PetscInt degree, PetscInt q_extra, SimpleBC bc, Physics phys) {
+PetscErrorCode SetUpDM(DM *dm, ProblemData problem, PetscInt degree, PetscInt q_extra, SimpleBC bc, Physics phys) {
   PetscInt  num_comp_q     = 5;
   DM        old_dm         = NULL;
   PetscBool SkipProjection = PETSC_FALSE;
@@ -116,24 +116,19 @@ PetscErrorCode SetUpDM(DM *dm, ProblemData *problem, PetscInt degree, PetscInt q
     DMLabel label;
     PetscCall(DMGetLabel(*dm, "Face Sets", &label));
     PetscCall(DMPlexLabelComplete(*dm, label));
-    // Set wall BCs
-    if (bc->num_wall > 0) {
-      PetscCall(DMAddBoundary(*dm, DM_BC_ESSENTIAL, "wall", label, bc->num_wall, bc->walls, 0, bc->num_comps, bc->wall_comps, NULL, NULL, NULL, NULL));
-    }
-    // Set symmetry BCs in the x direction
-    if (bc->num_symmetry[0] > 0) {
-      PetscInt comps[1] = {1};
-      PetscCall(DMAddBoundary(*dm, DM_BC_ESSENTIAL, "symmetry_x", label, bc->num_symmetry[0], bc->symmetries[0], 0, 1, comps, NULL, NULL, NULL, NULL));
-    }
-    // Set symmetry BCs in the y direction
-    if (bc->num_symmetry[1] > 0) {
-      PetscInt comps[1] = {2};
-      PetscCall(DMAddBoundary(*dm, DM_BC_ESSENTIAL, "symmetry_y", label, bc->num_symmetry[1], bc->symmetries[1], 0, 1, comps, NULL, NULL, NULL, NULL));
-    }
-    // Set symmetry BCs in the z direction
-    if (bc->num_symmetry[2] > 0) {
-      PetscInt comps[1] = {3};
-      PetscCall(DMAddBoundary(*dm, DM_BC_ESSENTIAL, "symmetry_z", label, bc->num_symmetry[2], bc->symmetries[2], 0, 1, comps, NULL, NULL, NULL, NULL));
+
+    for (PetscInt i = 0; i < problem->num_bc_defs; i++) {
+      BCDefinition    bc_def = problem->bc_defs[i];
+      PetscInt        num_essential_comps, num_label_values;
+      const PetscInt *essential_comps, *label_values;
+      const char     *name;
+
+      PetscCall(BCDefinitionGetEssential(bc_def, &num_essential_comps, &essential_comps));
+      if (essential_comps > 0) {
+        PetscCall(BCDefinitionGetInfo(bc_def, &name, &num_label_values, &label_values));
+        PetscCall(DMAddBoundary(*dm, DM_BC_ESSENTIAL, name, label, num_label_values, label_values, 0, num_essential_comps, essential_comps, NULL, NULL,
+                                NULL, NULL));
+      }
     }
     {
       PetscBool use_strongstg = PETSC_FALSE;

examples/fluids/src/setuplibceed.c

@@ -296,22 +296,14 @@ PetscErrorCode SetupLibceed(Ceed ceed, CeedData ceed_data, DM dm, User user, App
     PetscCallCeed(ceed, CeedQFunctionContextRestoreDataRead(problem->apply_vol_ifunction.qfunction_context, &gas));
   }
 
-  CeedElemRestriction elem_restr_jd_i;
-  CeedVector          jac_data;
-  CeedInt             num_qpts;
-  DMLabel             domain_label = NULL;
-  PetscInt            label_value = 0, height = 0, dm_field = 0;
-
-  // -----------------------------------------------------------------------------
-  // CEED Bases
-  // -----------------------------------------------------------------------------
-  DM dm_coord;
-
-  PetscCall(CreateBasisFromPlex(ceed, dm, domain_label, label_value, height, dm_field, &ceed_data->basis_q));
-  PetscCall(DMGetCoordinateDM(dm, &dm_coord));
-  PetscCall(CreateBasisFromPlex(ceed, dm_coord, domain_label, label_value, height, dm_field, &ceed_data->basis_x));
-  PetscCallCeed(ceed, CeedBasisCreateProjection(ceed_data->basis_x, ceed_data->basis_q, &ceed_data->basis_xc));
-  PetscCallCeed(ceed, CeedBasisGetNumQuadraturePoints(ceed_data->basis_q, &num_qpts));
+  {  // Create bases and element restrictions
+    DMLabel  domain_label = NULL;
+    PetscInt label_value = 0, height = 0, dm_field = 0;
+    DM       dm_coord;
+
+    PetscCall(DMGetCoordinateDM(dm, &dm_coord));
+    PetscCall(CreateBasisFromPlex(ceed, dm, domain_label, label_value, height, dm_field, &ceed_data->basis_q));
+    PetscCall(CreateBasisFromPlex(ceed, dm_coord, domain_label, label_value, height, dm_field, &ceed_data->basis_x));
 
     PetscCall(DMPlexCeedElemRestrictionCreate(ceed, dm, domain_label, label_value, height, 0, &ceed_data->elem_restr_q));
     PetscCall(DMPlexCeedElemRestrictionCoordinateCreate(ceed, dm, domain_label, label_value, height, &ceed_data->elem_restr_x));

examples/fluids/src/turb_spanstats.c

@@ -41,7 +41,6 @@ PetscErrorCode CreateStatsDM(User user, ProblemData problem, PetscInt degree) {
   // Get spanwise length
   PetscCall(DMGetBoundingBox(user->dm, domain_min, domain_max));
   user->spanstats.span_width = domain_max[2] - domain_min[2];
-  user->spanstats.span_width = domain_max[2] - domain_min[2];
 
   {  // Get DM from surface
     DM             parent_distributed_dm;
@@ -315,17 +314,8 @@ PetscErrorCode SetupL2ProjectionStats(Ceed ceed, User user, CeedData ceed_data,
 
   PetscCall(OperatorApplyContextCreate(NULL, user->spanstats.dm, ceed, op_proj_rhs, NULL, NULL, NULL, NULL, &user->spanstats.op_proj_rhs_ctx));
   PetscCall(CeedOperatorCreateLocalVecs(op_proj_rhs, DMReturnVecType(user->spanstats.dm), PETSC_COMM_SELF, &user->spanstats.Parent_Stats_loc, NULL));
-
-  // -- Setup LHS of L^2 projection
-  // Get q_data for mass matrix operator
-  PetscCallCeed(ceed, CeedOperatorCreate(ceed, ceed_data->qf_setup_sur, NULL, NULL, &op_setup_sur));
-  PetscCallCeed(ceed, CeedOperatorSetField(op_setup_sur, "dx", stats_data->elem_restr_parent_x, stats_data->basis_x, CEED_VECTOR_ACTIVE));
-  PetscCallCeed(ceed, CeedOperatorSetField(op_setup_sur, "weight", CEED_ELEMRESTRICTION_NONE, stats_data->basis_x, CEED_VECTOR_NONE));
-  PetscCallCeed(ceed, CeedOperatorSetField(op_setup_sur, "surface qdata", stats_data->elem_restr_parent_qd, CEED_BASIS_NONE, CEED_VECTOR_ACTIVE));
-  PetscCallCeed(ceed, CeedOperatorApply(op_setup_sur, stats_data->x_coord, stats_data->q_data, CEED_REQUEST_IMMEDIATE));
-
-  // CEED Restriction
-  PetscCallCeed(ceed, CeedElemRestrictionGetNumComponents(stats_data->elem_restr_parent_qd, &q_data_size));
+  PetscCall(QDataGet(ceed, user->spanstats.dm, domain_label, label_value, stats_data->elem_restr_parent_x, stats_data->basis_x, stats_data->x_coord,
+                     &elem_restr_qd, &q_data, &q_data_size));
 
   // Create Mass CeedOperator
   PetscCall(CreateMassQFunction(ceed, num_comp_stats, q_data_size, &qf_mass));
@@ -433,8 +423,8 @@ PetscErrorCode CreateStatisticCollectionOperator(Ceed ceed, User user, CeedData
   PetscCall(OperatorApplyContextCreate(user->dm, user->spanstats.dm, user->ceed, op_stats_collect, user->q_ceed, NULL, NULL, NULL,
                                        &user->spanstats.op_stats_collect_ctx));
 
-  PetscCall(CeedOperatorCreateLocalVecs(op_stats_collect, DMReturnVecType(user->spanstats.dm), PETSC_COMM_SELF, NULL,
-                                        &user->spanstats.Child_Stats_loc));
+  PetscCall(
+      CeedOperatorCreateLocalVecs(op_stats_collect, DMReturnVecType(user->spanstats.dm), PETSC_COMM_SELF, NULL, &user->spanstats.Child_Stats_loc));
   PetscCall(VecZeroEntries(user->spanstats.Child_Stats_loc));
 
   PetscCallCeed(ceed, CeedQFunctionDestroy(&qf_stats_collect));