Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Extend Expressions: Stats funcs #4368

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

Extend Expressions: Stats funcs #4368

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
06678dc
Create stats functions
lizdulac Sep 23, 2024
31b3887
wip shrink failing example
lizdulac Sep 24, 2024
c685331
Merge branch 'ornladios:master' into statsFuncs
lizdulac Oct 11, 2024
87f3693
change stats test to vector
lizdulac Oct 11, 2024
10b392a
Merge branch 'ornladios:master' into statsFuncs
lizdulac Oct 17, 2024
59c8e6b
Fix dimensions bug
lizdulac Oct 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 54 additions & 16 deletions source/adios2/toolkit/derived/Expression.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,26 +38,58 @@ const std::map<ExpressionOperator, OperatorProperty> op_property = {
{ExpressionOperator::OP_ATAN, {"ATAN", false}},
{ExpressionOperator::OP_MAGN, {"MAGNITUDE", false}},
{ExpressionOperator::OP_CROSS, {"CROSS", false}},
{ExpressionOperator::OP_CURL, {"CURL", false}}};
{ExpressionOperator::OP_CURL, {"CURL", false}},
{ExpressionOperator::OP_MIN, {"MIN", false}},
{ExpressionOperator::OP_MAX, {"MAX", false}},
{ExpressionOperator::OP_SUM, {"SUM", false}},
{ExpressionOperator::OP_MEAN, {"MEAN", false}},
{ExpressionOperator::OP_VARIANCE, {"VARIANCE", false}},
{ExpressionOperator::OP_STDEV, {"STDEV", false}}};

const std::map<std::string, ExpressionOperator> string_to_op = {
{"ALIAS", ExpressionOperator::OP_ALIAS}, /* Parser-use only */
{"PATH", ExpressionOperator::OP_PATH}, /* Parser-use only */
{"NUM", ExpressionOperator::OP_NUM}, /* Parser-use only */
{"INDEX", ExpressionOperator::OP_INDEX}, {"+", ExpressionOperator::OP_ADD},
{"add", ExpressionOperator::OP_ADD}, {"ADD", ExpressionOperator::OP_ADD},
{"-", ExpressionOperator::OP_SUBTRACT}, {"SUBTRACT", ExpressionOperator::OP_SUBTRACT},
{"/", ExpressionOperator::OP_DIV}, {"divide", ExpressionOperator::OP_DIV},
{"DIVIDE", ExpressionOperator::OP_DIV}, {"*", ExpressionOperator::OP_MULT},
{"multiply", ExpressionOperator::OP_MULT}, {"MULTIPLY", ExpressionOperator::OP_MULT},
{"SQRT", ExpressionOperator::OP_SQRT}, {"sqrt", ExpressionOperator::OP_SQRT},
{"pow", ExpressionOperator::OP_POW}, {"POW", ExpressionOperator::OP_POW},
{"sin", ExpressionOperator::OP_SIN}, {"cos", ExpressionOperator::OP_COS},
{"tan", ExpressionOperator::OP_TAN}, {"asin", ExpressionOperator::OP_ASIN},
{"acos", ExpressionOperator::OP_ACOS}, {"atan", ExpressionOperator::OP_ATAN},
{"^", ExpressionOperator::OP_POW}, {"magnitude", ExpressionOperator::OP_MAGN},
{"MAGNITUDE", ExpressionOperator::OP_MAGN}, {"cross", ExpressionOperator::OP_CROSS},
{"curl", ExpressionOperator::OP_CURL}, {"CURL", ExpressionOperator::OP_CURL}};
{"INDEX", ExpressionOperator::OP_INDEX},
{"+", ExpressionOperator::OP_ADD},
{"add", ExpressionOperator::OP_ADD},
{"ADD", ExpressionOperator::OP_ADD},
{"-", ExpressionOperator::OP_SUBTRACT},
{"SUBTRACT", ExpressionOperator::OP_SUBTRACT},
{"/", ExpressionOperator::OP_DIV},
{"divide", ExpressionOperator::OP_DIV},
{"DIVIDE", ExpressionOperator::OP_DIV},
{"*", ExpressionOperator::OP_MULT},
{"multiply", ExpressionOperator::OP_MULT},
{"MULTIPLY", ExpressionOperator::OP_MULT},
{"SQRT", ExpressionOperator::OP_SQRT},
{"sqrt", ExpressionOperator::OP_SQRT},
{"pow", ExpressionOperator::OP_POW},
{"POW", ExpressionOperator::OP_POW},
{"sin", ExpressionOperator::OP_SIN},
{"cos", ExpressionOperator::OP_COS},
{"tan", ExpressionOperator::OP_TAN},
{"asin", ExpressionOperator::OP_ASIN},
{"acos", ExpressionOperator::OP_ACOS},
{"atan", ExpressionOperator::OP_ATAN},
{"^", ExpressionOperator::OP_POW},
{"magnitude", ExpressionOperator::OP_MAGN},
{"MAGNITUDE", ExpressionOperator::OP_MAGN},
{"cross", ExpressionOperator::OP_CROSS},
{"curl", ExpressionOperator::OP_CURL},
{"CURL", ExpressionOperator::OP_CURL},
{"min", ExpressionOperator::OP_MIN},
{"MIN", ExpressionOperator::OP_MIN},
{"max", ExpressionOperator::OP_MAX},
{"MAX", ExpressionOperator::OP_MAX},
{"sum", ExpressionOperator::OP_SUM},
{"SUM", ExpressionOperator::OP_SUM},
{"mean", ExpressionOperator::OP_MEAN},
{"MEAN", ExpressionOperator::OP_MEAN},
{"variance", ExpressionOperator::OP_VARIANCE},
{"VARIANCE", ExpressionOperator::OP_VARIANCE},
{"stdev", ExpressionOperator::OP_STDEV},
{"STDEV", ExpressionOperator::OP_STDEV}};

inline std::string get_op_name(ExpressionOperator op) { return op_property.at(op).name; }

Expand Down Expand Up @@ -155,7 +187,13 @@ std::map<adios2::detail::ExpressionOperator, OperatorFunctions> OpFunctions = {
{adios2::detail::ExpressionOperator::OP_MAGN,
{MagnitudeFunc, SameDimsWithAgrFunc, SameTypeFunc}},
{adios2::detail::ExpressionOperator::OP_CROSS, {Cross3DFunc, Cross3DDimsFunc, SameTypeFunc}},
{adios2::detail::ExpressionOperator::OP_CURL, {Curl3DFunc, CurlDimsFunc, SameTypeFunc}}};
{adios2::detail::ExpressionOperator::OP_CURL, {Curl3DFunc, CurlDimsFunc, SameTypeFunc}},
{adios2::detail::ExpressionOperator::OP_MIN, {MinFunc, ScalarDimsFunc, SameTypeFunc}},
{adios2::detail::ExpressionOperator::OP_MAX, {MaxFunc, ScalarDimsFunc, SameTypeFunc}},
{adios2::detail::ExpressionOperator::OP_SUM, {SumFunc, ScalarDimsFunc, SameTypeFunc}},
{adios2::detail::ExpressionOperator::OP_MEAN, {MeanFunc, ScalarDimsFunc, SameTypeFunc}},
{adios2::detail::ExpressionOperator::OP_VARIANCE, {VarianceFunc, ScalarDimsFunc, SameTypeFunc}},
{adios2::detail::ExpressionOperator::OP_STDEV, {StDevFunc, ScalarDimsFunc, FloatTypeFunc}}};

Expression::Expression(std::string string_exp)
: m_Shape({0}), m_Start({0}), m_Count({0}), ExprString(string_exp)
Expand Down
9 changes: 8 additions & 1 deletion source/adios2/toolkit/derived/Expression.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,14 @@ enum ExpressionOperator
OP_ATAN,
OP_MAGN,
OP_CROSS,
OP_CURL
OP_CURL,
OP_MIN,
OP_MAX,
OP_SUM,
OP_MEAN,
OP_MEDIAN,
OP_VARIANCE,
OP_STDEV
};
}

Expand Down
186 changes: 182 additions & 4 deletions source/adios2/toolkit/derived/Function.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,18 @@ T *ApplyOneToOne(Iterator inputBegin, Iterator inputEnd, size_t dataSize,
return outValues;
}

template <class T>
T ApplyAllToOne(T *inputData, size_t dataSize, std::function<T(T, T)> compFct, T initVal = (T)0)
{
T outVal = initVal;
for (size_t i = 0; i < dataSize; i++)
{
T data = *(reinterpret_cast<T *>(inputData + i));
outVal = compFct(outVal, data);
}
return outVal;
}

template <class T>
T *AggregateOnLastDim(T *data, size_t dataSize, size_t nVariables, std::function<T(T, T)> compFct)
{
Expand Down Expand Up @@ -210,8 +222,8 @@ DerivedData MultFunc(std::vector<DerivedData> inputData, DataType type)
#define declare_type_mult(T) \
if (type == helper::GetDataType<T>()) \
{ \
T *multValues = detail::ApplyOneToOne<T>( \
inputData.begin(), inputData.end(), dataSize, [](T a, T b) { return a * b; }, 1); \
T *multValues = detail::ApplyOneToOne<T>(inputData.begin(), inputData.end(), dataSize, \
[](T a, T b) { return a * b; }, 1); \
return DerivedData({(void *)multValues, inputData[0].Start, inputData[0].Count}); \
}
ADIOS2_FOREACH_ATTRIBUTE_PRIMITIVE_STDTYPE_1ARG(declare_type_mult)
Expand All @@ -232,8 +244,8 @@ DerivedData DivFunc(std::vector<DerivedData> inputData, DataType type)
#define declare_type_div(T) \
if (type == helper::GetDataType<T>()) \
{ \
T *divValues = detail::ApplyOneToOne<T>( \
inputData.begin() + 1, inputData.end(), dataSize, [](T a, T b) { return a * b; }, 1); \
T *divValues = detail::ApplyOneToOne<T>(inputData.begin() + 1, inputData.end(), dataSize, \
[](T a, T b) { return a * b; }, 1); \
for (size_t i = 0; i < dataSize; i++) \
divValues[i] = *(reinterpret_cast<T *>(inputData[0].Data) + i) / divValues[i]; \
return DerivedData({(void *)divValues, inputData[0].Start, inputData[0].Count}); \
Expand Down Expand Up @@ -675,6 +687,166 @@ DerivedData Curl3DFunc(const std::vector<DerivedData> inputData, DataType type)
return DerivedData();
}

DerivedData MinFunc(std::vector<DerivedData> inputData, DataType type)
{
PERFSTUBS_SCOPED_TIMER("derived::Function::MinFunc");

#define declare_type_min(T) \
if (type == helper::GetDataType<T>()) \
{ \
T *minVal = (T *)malloc(sizeof(T)); \
*minVal = *((T *)(inputData[0].Data)); \
for (DerivedData d : inputData) \
{ \
size_t dataSize = std::accumulate(std::begin(d.Count), std::end(d.Count), 1, \
std::multiplies<size_t>()); \
*minVal = detail::ApplyAllToOne<T>((T *)d.Data, dataSize, \
[](T a, T b) { return std::min(a, b); }, *minVal); \
} \
return DerivedData({(void *)minVal, {0}, {1}}); \
}
ADIOS2_FOREACH_ATTRIBUTE_PRIMITIVE_STDTYPE_1ARG(declare_type_min)
helper::Throw<std::invalid_argument>("Derived", "Function", "MinFunc",
"Invalid variable types");
return DerivedData();
}

DerivedData MaxFunc(std::vector<DerivedData> inputData, DataType type)
{
PERFSTUBS_SCOPED_TIMER("derived::Function::MaxFunc");

#define declare_type_max(T) \
if (type == helper::GetDataType<T>()) \
{ \
T *maxVal = (T *)malloc(sizeof(T)); \
*maxVal = *((T *)(inputData[0].Data)); \
for (DerivedData d : inputData) \
{ \
size_t dataSize = std::accumulate(std::begin(d.Count), std::end(d.Count), 1, \
std::multiplies<size_t>()); \
*maxVal = detail::ApplyAllToOne<T>((T *)d.Data, dataSize, \
[](T a, T b) { return std::max(a, b); }, *maxVal); \
} \
return DerivedData({(void *)maxVal, {0}, {1}}); \
}
ADIOS2_FOREACH_ATTRIBUTE_PRIMITIVE_STDTYPE_1ARG(declare_type_max)
helper::Throw<std::invalid_argument>("Derived", "Function", "MaxFunc",
"Invalid variable types");
return DerivedData();
}

DerivedData SumFunc(std::vector<DerivedData> inputData, DataType type)
{
PERFSTUBS_SCOPED_TIMER("derived::Function::SumFunc");

#define declare_type_sum(T) \
if (type == helper::GetDataType<T>()) \
{ \
T *sumVal = (T *)malloc(sizeof(T)); \
*sumVal = (T)0; \
for (DerivedData d : inputData) \
{ \
size_t dataSize = std::accumulate(std::begin(d.Count), std::end(d.Count), 1, \
std::multiplies<size_t>()); \
*sumVal = detail::ApplyAllToOne<T>((T *)inputData[0].Data, dataSize, \
[](T a, T b) { return a + b; }, *sumVal); \
} \
return DerivedData({(void *)sumVal, {0}, {1}}); \
}
ADIOS2_FOREACH_ATTRIBUTE_PRIMITIVE_STDTYPE_1ARG(declare_type_sum)
helper::Throw<std::invalid_argument>("Derived", "Function", "SumFunc",
"Invalid variable types");
return DerivedData();
}

DerivedData MeanFunc(std::vector<DerivedData> inputData, DataType type)
{
PERFSTUBS_SCOPED_TIMER("derived::Function::MeanFunc");

#define declare_type_mean(T) \
if (type == helper::GetDataType<T>()) \
{ \
T *meanVal = (T *)malloc(sizeof(T)); \
*meanVal = (T)0; \
size_t totalSize = 0; \
for (DerivedData d : inputData) \
{ \
size_t dataSize = std::accumulate(std::begin(d.Count), std::end(d.Count), 1, \
std::multiplies<size_t>()); \
totalSize += dataSize; \
*meanVal = detail::ApplyAllToOne<T>((T *)inputData[0].Data, dataSize, \
[](T a, T b) { return a + b; }, *meanVal); \
} \
*meanVal /= (T)totalSize; \
return DerivedData({(void *)meanVal, {0}, {1}}); \
}
ADIOS2_FOREACH_ATTRIBUTE_PRIMITIVE_STDTYPE_1ARG(declare_type_mean)
helper::Throw<std::invalid_argument>("Derived", "Function", "MeanFunc",
"Invalid variable types");
return DerivedData();
}

DerivedData VarianceFunc(std::vector<DerivedData> inputData, DataType type)
{
PERFSTUBS_SCOPED_TIMER("derived::Function::VarianceFunc");
DerivedData meanData = MeanFunc(inputData, type);
size_t totalSize = 0;
for (DerivedData d : inputData)
totalSize +=
std::accumulate(std::begin(d.Count), std::end(d.Count), 1, std::multiplies<size_t>());

#define declare_type_variance(T) \
if (type == helper::GetDataType<T>()) \
{ \
T meanVal = *((T *)meanData.Data); \
T *varianceVal = (T *)malloc(sizeof(T)); \
*varianceVal = (T)0; \
for (DerivedData d : inputData) \
{ \
size_t dataSize = std::accumulate(std::begin(d.Count), std::end(d.Count), 1, \
std::multiplies<size_t>()); \
*varianceVal = detail::ApplyAllToOne<T>( \
(T *)inputData[0].Data, dataSize, \
[&meanVal, &totalSize](T a, T b) { \
return a + (((b - meanVal) * (b - meanVal)) / (T)totalSize); \
}, \
*varianceVal); \
} \
return DerivedData({(void *)varianceVal, {0}, {1}}); \
}
ADIOS2_FOREACH_ATTRIBUTE_PRIMITIVE_STDTYPE_1ARG(declare_type_variance)
helper::Throw<std::invalid_argument>("Derived", "Function", "VarianceFunc",
"Invalid variable types");
return DerivedData();
}

DerivedData StDevFunc(std::vector<DerivedData> inputData, DataType type)
{
PERFSTUBS_SCOPED_TIMER("derived::Function::StdevFunc");
DataType inputType = inputData[0].Type;
DerivedData varianceData = VarianceFunc(inputData, inputType);

if (inputType == DataType::LongDouble)
{
long double varianceVal = *((long double *)varianceData.Data);
long double *stdevVal = (long double *)malloc(sizeof(long double));
*stdevVal = std::sqrt(varianceVal);
return DerivedData({(void *)stdevVal, {0}, {1}});
}
#define declare_type_stdev(T) \
else if (inputType == helper::GetDataType<T>()) \
{ \
T varianceVal = *((T *)varianceData.Data); \
double *stdevVal = (double *)malloc(sizeof(double)); \
*stdevVal = std::sqrt(varianceVal); \
return DerivedData({(void *)stdevVal, {0}, {1}}); \
}
ADIOS2_FOREACH_ATTRIBUTE_PRIMITIVE_STDTYPE_1ARG(declare_type_stdev)
helper::Throw<std::invalid_argument>("Derived", "Function", "StdevFunc",
"Invalid variable types");
return DerivedData();
}

/* Functions that return output dimensions
* Input: A list of variable dimensions (start, count, shape)
* Output: (start, count, shape) of the output operation */
Expand Down Expand Up @@ -752,6 +924,12 @@ std::tuple<Dims, Dims, Dims> CurlDimsFunc(std::vector<std::tuple<Dims, Dims, Dim
return output;
}

std::tuple<Dims, Dims, Dims> ScalarDimsFunc(std::vector<std::tuple<Dims, Dims, Dims>> input)
{
// Start, Count, Shape
return {{0}, {1}, {1}};
}

DataType SameTypeFunc(DataType input) { return input; }

DataType FloatTypeFunc(DataType input)
Expand Down
Loading
Loading