diff --git a/CMakeLists.txt b/CMakeLists.txt index 44d100e444..eabfc055ee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -376,6 +376,7 @@ if (zoidberg_FOUND EQUAL 0) else() set(zoidberg_FOUND OFF) endif() +message(STATUS "Found Zoidberg for FCI tests: ${zoidberg_FOUND}") option(BOUT_GENERATE_FIELDOPS "Automatically re-generate the Field arithmetic operators from the Python templates. \ Requires Python3, clang-format, and Jinja2. Turn this OFF to skip generating them if, for example, \ diff --git a/include/bout/difops.hxx b/include/bout/difops.hxx index 71053d454a..c2fdac195d 100644 --- a/include/bout/difops.hxx +++ b/include/bout/difops.hxx @@ -40,7 +40,9 @@ #include "bout/field3d.hxx" #include "bout/bout_types.hxx" -#include "bout/solver.hxx" +#include "bout/coordinates.hxx" + +class Solver; /*! * Parallel derivative (central differencing) in Y diff --git a/include/bout/interpolation_xz.hxx b/include/bout/interpolation_xz.hxx index 6c7419f7e4..4dd24259fd 100644 --- a/include/bout/interpolation_xz.hxx +++ b/include/bout/interpolation_xz.hxx @@ -24,7 +24,9 @@ #ifndef BOUT_INTERP_XZ_H #define BOUT_INTERP_XZ_H -#include "bout/mask.hxx" +#include +#include +#include #define USE_NEW_WEIGHTS 1 #if BOUT_HAS_PETSC @@ -166,7 +168,8 @@ protected: #endif public: - XZHermiteSpline(Mesh* mesh = nullptr) : XZHermiteSpline(0, mesh) {} + XZHermiteSpline(Mesh* mesh = nullptr, [[maybe_unused]] Options* options = nullptr) + : XZHermiteSpline(0, mesh) {} XZHermiteSpline(int y_offset = 0, Mesh* mesh = nullptr); XZHermiteSpline(const BoutMask& mask, int y_offset = 0, Mesh* mesh = nullptr) : XZHermiteSpline(y_offset, mesh) { @@ -210,9 +213,29 @@ public: /// but also degrades accuracy near maxima and minima. /// Perhaps should only impose near boundaries, since that is where /// problems most obviously occur. +/// +/// You can control how tight the clipping to the range of the neighbouring cell +/// values through ``rtol`` and ``atol``: +/// +/// diff = (max_of_neighours - min_of_neighours) * rtol + atol +/// +/// and the interpolated value is instead clipped to the range +/// ``[min_of_neighours - diff, max_of_neighours + diff]`` class XZMonotonicHermiteSpline : public XZHermiteSpline { + /// Absolute tolerance for clipping + BoutReal atol = 0.0; + /// Relative tolerance for clipping + BoutReal rtol = 1.0; + public: - XZMonotonicHermiteSpline(Mesh* mesh = nullptr) : XZHermiteSpline(0, mesh) { + XZMonotonicHermiteSpline(Mesh* mesh = nullptr, Options* options = nullptr) + : XZHermiteSpline(0, mesh), + atol{(*options)["atol"] + .doc("Absolute tolerance for clipping overshoot") + .withDefault(0.0)}, + rtol{(*options)["rtol"] + .doc("Relative tolerance for clipping overshoot") + .withDefault(1.0)} { if (localmesh->getNXPE() > 1) { throw BoutException("Do not support MPI splitting in X"); } @@ -248,7 +271,8 @@ class XZLagrange4pt : public XZInterpolation { Field3D t_x, t_z; public: - XZLagrange4pt(Mesh* mesh = nullptr) : XZLagrange4pt(0, mesh) {} + XZLagrange4pt(Mesh* mesh = nullptr, [[maybe_unused]] Options* options = nullptr) + : XZLagrange4pt(0, mesh) {} XZLagrange4pt(int y_offset = 0, Mesh* mesh = nullptr); XZLagrange4pt(const BoutMask& mask, int y_offset = 0, Mesh* mesh = nullptr) : XZLagrange4pt(y_offset, mesh) { @@ -284,7 +308,8 @@ class XZBilinear : public XZInterpolation { Field3D w0, w1, w2, w3; public: - XZBilinear(Mesh* mesh = nullptr) : XZBilinear(0, mesh) {} + XZBilinear(Mesh* mesh = nullptr, [[maybe_unused]] Options* options = nullptr) + : XZBilinear(0, mesh) {} XZBilinear(int y_offset = 0, Mesh* mesh = nullptr); XZBilinear(const BoutMask& mask, int y_offset = 0, Mesh* mesh = nullptr) : XZBilinear(y_offset, mesh) { @@ -308,7 +333,7 @@ public: }; class XZInterpolationFactory - : public Factory { + : public Factory { public: static constexpr auto type_name = "XZInterpolation"; static constexpr auto section_name = "xzinterpolation"; @@ -316,10 +341,10 @@ public: static constexpr auto default_type = "hermitespline"; ReturnType create(Options* options = nullptr, Mesh* mesh = nullptr) const { - return Factory::create(getType(options), mesh); + return Factory::create(getType(options), mesh, options); } - ReturnType create(const std::string& type, [[maybe_unused]] Options* options) const { - return Factory::create(type, nullptr); + ReturnType create(const std::string& type, Options* options) const { + return Factory::create(type, nullptr, options); } static void ensureRegistered(); diff --git a/src/mesh/coordinates.cxx b/src/mesh/coordinates.cxx index 3dfee6a553..12e465ffb6 100644 --- a/src/mesh/coordinates.cxx +++ b/src/mesh/coordinates.cxx @@ -1577,7 +1577,7 @@ Field3D Coordinates::Div_par(const Field3D& f, CELL_LOC outloc, // Need Bxy at location of f, which might be different from location of this // Coordinates object - auto Bxy_floc = f.getCoordinates()->Bxy; + const auto& Bxy_floc = f.getCoordinates()->Bxy; if (!f.hasParallelSlices()) { // No yup/ydown fields. The Grad_par operator will diff --git a/src/mesh/interpolation/hermite_spline_xz.cxx b/src/mesh/interpolation/hermite_spline_xz.cxx index 5020a5b9a3..27a4f1d614 100644 --- a/src/mesh/interpolation/hermite_spline_xz.cxx +++ b/src/mesh/interpolation/hermite_spline_xz.cxx @@ -171,7 +171,19 @@ void XZHermiteSpline::calcWeights(const Field3D& delta_x, const Field3D& delta_z BoutReal t_x = delta_x(x, y, z) - static_cast(i_corn); BoutReal t_z = delta_z(x, y, z) - static_cast(k_corner(x, y, z)); - // NOTE: A (small) hack to avoid one-sided differences + // NOTE: A (small) hack to avoid one-sided differences. We need at + // least 2 interior points due to an awkwardness with the + // boundaries. The splines need derivatives in x, but we don't + // know the value in the boundaries, so _any_ interpolation in the + // last interior cell can't be done. Instead, we fudge the + // interpolation in the last cell to be at the extreme right-hand + // edge of the previous cell (that is, exactly on the last + // interior point). However, this doesn't work with only one + // interior point, because we have to do something similar to the + // _first_ cell, and these two fudges cancel out and we end up + // indexing into the boundary anyway. + // TODO(peter): Can we remove this if we apply (dirichlet?) BCs to + // the X derivatives? Note that we need at least _2_ if (i_corn >= xend) { i_corn = xend - 1; t_x = 1.0; diff --git a/src/mesh/interpolation/monotonic_hermite_spline_xz.cxx b/src/mesh/interpolation/monotonic_hermite_spline_xz.cxx index f23bfd499e..f206ed1e0f 100644 --- a/src/mesh/interpolation/monotonic_hermite_spline_xz.cxx +++ b/src/mesh/interpolation/monotonic_hermite_spline_xz.cxx @@ -25,7 +25,7 @@ #include "bout/interpolation_xz.hxx" #include "bout/mesh.hxx" -#include +#include Field3D XZMonotonicHermiteSpline::interpolate(const Field3D& f, const std::string& region) const { @@ -80,7 +80,6 @@ Field3D XZMonotonicHermiteSpline::interpolate(const Field3D& f, // Perhaps should only impose near boundaries, since that is where // problems most obviously occur. const BoutReal localmax = BOUTMAX(f[ic], f[icxp], f[iczp], f[icxpzp]); - const BoutReal localmin = BOUTMIN(f[ic], f[icxp], f[iczp], f[icxpzp]); ASSERT2(std::isfinite(localmax) || i.x() < localmesh->xstart @@ -88,12 +87,10 @@ Field3D XZMonotonicHermiteSpline::interpolate(const Field3D& f, ASSERT2(std::isfinite(localmin) || i.x() < localmesh->xstart || i.x() > localmesh->xend); - if (result > localmax) { - result = localmax; - } - if (result < localmin) { - result = localmin; - } + const auto diff = ((localmax - localmin) * rtol) + atol; + + result = std::min(result, localmax + diff); + result = std::max(result, localmin - diff); f_interp[iyp] = result; } diff --git a/src/mesh/parallel/fci.cxx b/src/mesh/parallel/fci.cxx index e8d3af1cdb..6243bbb67a 100644 --- a/src/mesh/parallel/fci.cxx +++ b/src/mesh/parallel/fci.cxx @@ -37,79 +37,100 @@ **************************************************************************/ #include "fci.hxx" + +#include "bout/assert.hxx" +#include "bout/bout_types.hxx" +#include "bout/boutexception.hxx" +#include "bout/field2d.hxx" +#include "bout/field3d.hxx" +#include "bout/field_data.hxx" +#include "bout/mesh.hxx" +#include "bout/msg_stack.hxx" +#include "bout/options.hxx" #include "bout/parallel_boundary_op.hxx" #include "bout/parallel_boundary_region.hxx" -#include -#include -#include -#include -#include +#include "bout/paralleltransform.hxx" +#include "bout/region.hxx" + +#include +#include +#include +#include +#include +#include #include +#include + +using namespace std::string_view_literals; -FCIMap::FCIMap(Mesh& mesh, const Coordinates::FieldMetric& UNUSED(dy), Options& options, - int offset_, const std::shared_ptr& inner_boundary, +FCIMap::FCIMap(Mesh& mesh, [[maybe_unused]] const Coordinates::FieldMetric& dy, + Options& options, int offset, + const std::shared_ptr& inner_boundary, const std::shared_ptr& outer_boundary, bool zperiodic) - : map_mesh(mesh), offset(offset_), - region_no_boundary(map_mesh.getRegion("RGN_NOBNDRY")), + : map_mesh(&mesh), offset_(offset), + region_no_boundary(map_mesh->getRegion("RGN_NOBNDRY")), corner_boundary_mask(map_mesh) { - TRACE("Creating FCIMAP for direction {:d}", offset); + TRACE("Creating FCIMAP for direction {:d}", offset_); - if (offset == 0) { + if (offset_ == 0) { throw BoutException( "FCIMap called with offset = 0; You probably didn't mean to do that"); } auto& interpolation_options = options["xzinterpolation"]; - interp = - XZInterpolationFactory::getInstance().create(&interpolation_options, &map_mesh); - interp->setYOffset(offset); + interp = XZInterpolationFactory::getInstance().create(&interpolation_options, map_mesh); + interp->setYOffset(offset_); interp_corner = - XZInterpolationFactory::getInstance().create(&interpolation_options, &map_mesh); - interp_corner->setYOffset(offset); + XZInterpolationFactory::getInstance().create(&interpolation_options, map_mesh); + interp_corner->setYOffset(offset_); // Index-space coordinates of forward/backward points - Field3D xt_prime{&map_mesh}, zt_prime{&map_mesh}; + Field3D xt_prime{map_mesh}; + Field3D zt_prime{map_mesh}; // Real-space coordinates of grid points - Field3D R{&map_mesh}, Z{&map_mesh}; + Field3D R{map_mesh}; + Field3D Z{map_mesh}; // Real-space coordinates of forward/backward points - Field3D R_prime{&map_mesh}, Z_prime{&map_mesh}; + Field3D R_prime{map_mesh}; + Field3D Z_prime{map_mesh}; - map_mesh.get(R, "R", 0.0, false); - map_mesh.get(Z, "Z", 0.0, false); + map_mesh->get(R, "R", 0.0, false); + map_mesh->get(Z, "Z", 0.0, false); // Get a unique name for a field based on the sign/magnitude of the offset - const auto parallel_slice_field_name = [&](std::string field) -> std::string { - const std::string direction = (offset > 0) ? "forward" : "backward"; + const auto parallel_slice_field_name = [&](std::string_view field) -> std::string { + const auto direction = (offset_ > 0) ? "forward"sv : "backward"sv; // We only have a suffix for parallel slices beyond the first // This is for backwards compatibility - const std::string slice_suffix = - (std::abs(offset) > 1) ? "_" + std::to_string(std::abs(offset)) : ""; - return direction + "_" + field + slice_suffix; + if (std::abs(offset_) == 1) { + return fmt::format("{}_{}", direction, field); + } + return fmt::format("{}_{}_{}", direction, field, std::abs(offset_)); }; // If we can't read in any of these fields, things will silently not // work, so best throw - if (map_mesh.get(xt_prime, parallel_slice_field_name("xt_prime"), 0.0, false) != 0) { + if (map_mesh->get(xt_prime, parallel_slice_field_name("xt_prime"), 0.0, false) != 0) { throw BoutException("Could not read {:s} from grid file!\n" " Either add it to the grid file, or reduce MYG", parallel_slice_field_name("xt_prime")); } - if (map_mesh.get(zt_prime, parallel_slice_field_name("zt_prime"), 0.0, false) != 0) { + if (map_mesh->get(zt_prime, parallel_slice_field_name("zt_prime"), 0.0, false) != 0) { throw BoutException("Could not read {:s} from grid file!\n" " Either add it to the grid file, or reduce MYG", parallel_slice_field_name("zt_prime")); } - if (map_mesh.get(R_prime, parallel_slice_field_name("R"), 0.0, false) != 0) { + if (map_mesh->get(R_prime, parallel_slice_field_name("R"), 0.0, false) != 0) { throw BoutException("Could not read {:s} from grid file!\n" " Either add it to the grid file, or reduce MYG", parallel_slice_field_name("R")); } - if (map_mesh.get(Z_prime, parallel_slice_field_name("Z"), 0.0, false) != 0) { + if (map_mesh->get(Z_prime, parallel_slice_field_name("Z"), 0.0, false) != 0) { throw BoutException("Could not read {:s} from grid file!\n" " Either add it to the grid file, or reduce MYG", parallel_slice_field_name("Z")); @@ -157,25 +178,26 @@ FCIMap::FCIMap(Mesh& mesh, const Coordinates::FieldMetric& UNUSED(dy), Options& interp->calcWeights(xt_prime, zt_prime); } - const int ncz = map_mesh.LocalNz; + const int ncz = map_mesh->LocalNz; BoutMask to_remove(map_mesh); - const int xend = - map_mesh.xstart + (map_mesh.xend - map_mesh.xstart + 1) * map_mesh.getNXPE() - 1; + const int xend = map_mesh->xstart + + ((map_mesh->xend - map_mesh->xstart + 1) * map_mesh->getNXPE()) - 1; // Serial loop because call to BoundaryRegionPar::addPoint // (probably?) can't be done in parallel BOUT_FOR_SERIAL(i, xt_prime.getRegion("RGN_NOBNDRY")) { // z is periodic, so make sure the z-index wraps around if (zperiodic) { - zt_prime[i] = zt_prime[i] - - ncz * (static_cast(zt_prime[i] / static_cast(ncz))); + zt_prime[i] = + zt_prime[i] + - (ncz * (static_cast(zt_prime[i] / static_cast(ncz)))); if (zt_prime[i] < 0.0) { zt_prime[i] += ncz; } } - if ((xt_prime[i] >= map_mesh.xstart) and (xt_prime[i] <= xend)) { + if ((xt_prime[i] >= map_mesh->xstart) and (xt_prime[i] <= xend)) { // Not a boundary continue; } @@ -215,7 +237,7 @@ FCIMap::FCIMap(Mesh& mesh, const Coordinates::FieldMetric& UNUSED(dy), Options& const BoutReal dR_dz = 0.5 * (R[i_zp] - R[i_zm]); const BoutReal dZ_dz = 0.5 * (Z[i_zp] - Z[i_zm]); - const BoutReal det = dR_dx * dZ_dz - dR_dz * dZ_dx; // Determinant of 2x2 matrix + const BoutReal det = (dR_dx * dZ_dz) - (dR_dz * dZ_dx); // Determinant of 2x2 matrix const BoutReal dR = R_prime[i] - R[i]; const BoutReal dZ = Z_prime[i] - Z[i]; @@ -228,9 +250,9 @@ FCIMap::FCIMap(Mesh& mesh, const Coordinates::FieldMetric& UNUSED(dy), Options& // outer boundary. However, if any of the surrounding points are negative, // that also means inner. So to differentiate between inner and outer we // need at least 2 points in the domain. - ASSERT2(map_mesh.xend - map_mesh.xstart >= 2); - auto boundary = (xt_prime[i] < map_mesh.xstart) ? inner_boundary : outer_boundary; - boundary->add_point(x, y, z, x + dx, y + 0.5 * offset, + ASSERT2(map_mesh->xend - map_mesh->xstart >= 2); + auto boundary = (xt_prime[i] < map_mesh->xstart) ? inner_boundary : outer_boundary; + boundary->add_point(x, y, z, x + dx, y + (0.5 * offset_), z + dz, // Intersection point in local index space 0.5, // Distance to intersection 1 // Default to that there is a point in the other direction @@ -240,13 +262,14 @@ FCIMap::FCIMap(Mesh& mesh, const Coordinates::FieldMetric& UNUSED(dy), Options& interp->setRegion(region_no_boundary); - const auto region = fmt::format("RGN_YPAR_{:+d}", offset); - if (not map_mesh.hasRegion3D(region)) { + const auto region = fmt::format("RGN_YPAR_{:+d}", offset_); + if (not map_mesh->hasRegion3D(region)) { // The valid region for this slice - map_mesh.addRegion3D( - region, Region(map_mesh.xstart, map_mesh.xend, map_mesh.ystart + offset, - map_mesh.yend + offset, 0, map_mesh.LocalNz - 1, - map_mesh.LocalNy, map_mesh.LocalNz)); + map_mesh->addRegion3D(region, Region(map_mesh->xstart, map_mesh->xend, + map_mesh->ystart + offset_, + map_mesh->yend + offset_, 0, + map_mesh->LocalNz - 1, map_mesh->LocalNy, + map_mesh->LocalNz)); } } @@ -254,7 +277,7 @@ Field3D FCIMap::integrate(Field3D& f) const { TRACE("FCIMap::integrate"); ASSERT1(f.getDirectionY() == YDirectionType::Standard); - ASSERT1(&map_mesh == f.getMesh()); + ASSERT1(map_mesh == f.getMesh()); // Cell centre values Field3D centre = interp->interpolate(f); @@ -269,7 +292,7 @@ Field3D FCIMap::integrate(Field3D& f) const { #endif BOUT_FOR(i, region_no_boundary) { - const auto inext = i.yp(offset); + const auto inext = i.yp(offset_); const BoutReal f_c = centre[inext]; const auto izm = i.zm(); const int x = i.x(); @@ -278,7 +301,7 @@ Field3D FCIMap::integrate(Field3D& f) const { const int zm = izm.z(); if (corner_boundary_mask(x, y, z) || corner_boundary_mask(x - 1, y, z) || corner_boundary_mask(x, y, zm) || corner_boundary_mask(x - 1, y, zm) - || (x == map_mesh.xstart)) { + || (x == map_mesh->xstart)) { // One of the corners leaves the domain. // Use the cell centre value, since boundary conditions are not // currently applied to corners. @@ -299,19 +322,70 @@ Field3D FCIMap::integrate(Field3D& f) const { return result; } +FCITransform::FCITransform(Mesh& mesh, const Coordinates::FieldMetric& dy, bool zperiodic, + Options* opt) + : ParallelTransform(mesh, opt), R{&mesh}, Z{&mesh} { + + // check the coordinate system used for the grid data source + FCITransform::checkInputGrid(); + + // Real-space coordinates of grid cells + mesh.get(R, "R", 0.0, false); + mesh.get(Z, "Z", 0.0, false); + + auto forward_boundary_xin = + std::make_shared("FCI_forward", BNDRY_PAR_FWD_XIN, +1, &mesh); + auto backward_boundary_xin = + std::make_shared("FCI_backward", BNDRY_PAR_BKWD_XIN, -1, &mesh); + auto forward_boundary_xout = + std::make_shared("FCI_forward", BNDRY_PAR_FWD_XOUT, +1, &mesh); + auto backward_boundary_xout = + std::make_shared("FCI_backward", BNDRY_PAR_BKWD_XOUT, -1, &mesh); + + // Add the boundary region to the mesh's vector of parallel boundaries + mesh.addBoundaryPar(forward_boundary_xin, BoundaryParType::xin_fwd); + mesh.addBoundaryPar(backward_boundary_xin, BoundaryParType::xin_bwd); + mesh.addBoundaryPar(forward_boundary_xout, BoundaryParType::xout_fwd); + mesh.addBoundaryPar(backward_boundary_xout, BoundaryParType::xout_bwd); + + field_line_maps.reserve(static_cast(mesh.ystart) * 2); + for (int offset = 1; offset < mesh.ystart + 1; ++offset) { + field_line_maps.emplace_back(mesh, dy, options, offset, forward_boundary_xin, + forward_boundary_xout, zperiodic); + field_line_maps.emplace_back(mesh, dy, options, -offset, backward_boundary_xin, + backward_boundary_xout, zperiodic); + } + ASSERT0(mesh.ystart == 1); + const std::array bndries = {forward_boundary_xin, forward_boundary_xout, + backward_boundary_xin, backward_boundary_xout}; + for (const auto& bndry : bndries) { + for (const auto& bndry2 : bndries) { + if (bndry->dir == bndry2->dir) { + continue; + } + for (bndry->first(); !bndry->isDone(); bndry->next()) { + if (bndry2->contains(*bndry)) { + bndry->setValid(0); + } + } + } + } +} + void FCITransform::checkInputGrid() { std::string parallel_transform; if (mesh.isDataSourceGridFile() - && !mesh.get(parallel_transform, "parallel_transform")) { + && (mesh.get(parallel_transform, "parallel_transform") == 0)) { if (parallel_transform != "fci") { throw BoutException( "Incorrect parallel transform type '" + parallel_transform + "' used " "to generate metric components for FCITransform. Should be 'fci'."); } - } // else: parallel_transform variable not found in grid input, indicates older input - // file or grid from options so must rely on the user having ensured the type is - // correct + } + // else: parallel_transform variable not found in grid input, indicates older input + // file or grid from options so must rely on the user having ensured the type is + // correct } void FCITransform::calcParallelSlices(Field3D& f) { @@ -327,8 +401,8 @@ void FCITransform::calcParallelSlices(Field3D& f) { // Interpolate f onto yup and ydown fields for (const auto& map : field_line_maps) { - f.ynext(map.offset) = map.interpolate(f); - f.ynext(map.offset).setRegion(fmt::format("RGN_YPAR_{:+d}", map.offset)); + f.ynext(map.offset()) = map.interpolate(f); + f.ynext(map.offset()).setRegion(fmt::format("RGN_YPAR_{:+d}", map.offset())); } } @@ -345,7 +419,7 @@ void FCITransform::integrateParallelSlices(Field3D& f) { // Integrate f onto yup and ydown fields for (const auto& map : field_line_maps) { - f.ynext(map.offset) = map.integrate(f); + f.ynext(map.offset()) = map.integrate(f); } } diff --git a/src/mesh/parallel/fci.hxx b/src/mesh/parallel/fci.hxx index 1a02f558e1..65529a4c4e 100644 --- a/src/mesh/parallel/fci.hxx +++ b/src/mesh/parallel/fci.hxx @@ -26,6 +26,11 @@ #ifndef BOUT_FCITRANSFORM_H #define BOUT_FCITRANSFORM_H +#include "bout/assert.hxx" +#include "bout/bout_types.hxx" +#include "bout/boutexception.hxx" +#include "bout/coordinates.hxx" +#include "bout/region.hxx" #include #include #include @@ -33,25 +38,26 @@ #include #include +#include #include +class BoundaryRegionPar; +class FieldPerp; +class Field2D; +class Field3D; +class Options; + /// Field line map - contains the coefficients for interpolation class FCIMap { /// Interpolation objects std::unique_ptr interp; // Cell centre std::unique_ptr interp_corner; // Cell corner at (x+1, z+1) -public: - FCIMap() = delete; - FCIMap(Mesh& mesh, const Coordinates::FieldMetric& dy, Options& options, int offset, - const std::shared_ptr& inner_boundary, - const std::shared_ptr& outer_boundary, bool zperiodic); - // The mesh this map was created on - Mesh& map_mesh; + Mesh* map_mesh; /// Direction of map - const int offset; + int offset_; /// region containing all points where the field line has not left the /// domain @@ -59,8 +65,17 @@ public: /// If any of the integration area has left the domain BoutMask corner_boundary_mask; +public: + FCIMap() = delete; + FCIMap(Mesh& mesh, const Coordinates::FieldMetric& dy, Options& options, int offset, + const std::shared_ptr& inner_boundary, + const std::shared_ptr& outer_boundary, bool zperiodic); + + /// Direction of map + int offset() const { return offset_; } + Field3D interpolate(Field3D& f) const { - ASSERT1(&map_mesh == f.getMesh()); + ASSERT1(map_mesh == f.getMesh()); return interp->interpolate(f); } @@ -72,55 +87,7 @@ class FCITransform : public ParallelTransform { public: FCITransform() = delete; FCITransform(Mesh& mesh, const Coordinates::FieldMetric& dy, bool zperiodic = true, - Options* opt = nullptr) - : ParallelTransform(mesh, opt), R{&mesh}, Z{&mesh} { - - // check the coordinate system used for the grid data source - FCITransform::checkInputGrid(); - - // Real-space coordinates of grid cells - mesh.get(R, "R", 0.0, false); - mesh.get(Z, "Z", 0.0, false); - - auto forward_boundary_xin = - std::make_shared("FCI_forward", BNDRY_PAR_FWD_XIN, +1, &mesh); - auto backward_boundary_xin = std::make_shared( - "FCI_backward", BNDRY_PAR_BKWD_XIN, -1, &mesh); - auto forward_boundary_xout = - std::make_shared("FCI_forward", BNDRY_PAR_FWD_XOUT, +1, &mesh); - auto backward_boundary_xout = std::make_shared( - "FCI_backward", BNDRY_PAR_BKWD_XOUT, -1, &mesh); - - // Add the boundary region to the mesh's vector of parallel boundaries - mesh.addBoundaryPar(forward_boundary_xin, BoundaryParType::xin_fwd); - mesh.addBoundaryPar(backward_boundary_xin, BoundaryParType::xin_bwd); - mesh.addBoundaryPar(forward_boundary_xout, BoundaryParType::xout_fwd); - mesh.addBoundaryPar(backward_boundary_xout, BoundaryParType::xout_bwd); - - field_line_maps.reserve(mesh.ystart * 2); - for (int offset = 1; offset < mesh.ystart + 1; ++offset) { - field_line_maps.emplace_back(mesh, dy, options, offset, forward_boundary_xin, - forward_boundary_xout, zperiodic); - field_line_maps.emplace_back(mesh, dy, options, -offset, backward_boundary_xin, - backward_boundary_xout, zperiodic); - } - ASSERT0(mesh.ystart == 1); - std::shared_ptr bndries[]{ - forward_boundary_xin, forward_boundary_xout, backward_boundary_xin, - backward_boundary_xout}; - for (auto& bndry : bndries) { - for (const auto& bndry2 : bndries) { - if (bndry->dir == bndry2->dir) { - continue; - } - for (bndry->first(); !bndry->isDone(); bndry->next()) { - if (bndry2->contains(*bndry)) { - bndry->setValid(0); - } - } - } - } - } + Options* opt = nullptr); void calcParallelSlices(Field3D& f) override; diff --git a/tests/MMS/spatial/fci/data/BOUT.inp b/tests/MMS/spatial/fci/data/BOUT.inp index 5f2001a906..93e2101473 100644 --- a/tests/MMS/spatial/fci/data/BOUT.inp +++ b/tests/MMS/spatial/fci/data/BOUT.inp @@ -1,15 +1,17 @@ - input_field = sin(y - 2*z) + sin(y - z) - -solution = (6.28318530717959*(0.01*x + 0.045)*(-2*cos(y - 2*z) - cos(y - z)) + 0.628318530717959*cos(y - 2*z) + 0.628318530717959*cos(y - z))/sqrt((0.01*x + 0.045)^2 + 1.0) - -MXG = 1 -MYG = 1 -NXPE = 1 +grad_par_solution = (6.28318530717959*(0.01*x + 0.045)*(-2*cos(y - 2*z) - cos(y - z)) + 0.628318530717959*cos(y - 2*z) + 0.628318530717959*cos(y - z))/sqrt((0.01*x + 0.045)^2 + 1.0) +grad2_par2_solution = (6.28318530717959*(0.01*x + 0.045)*(6.28318530717959*(0.01*x + 0.045)*(-4*sin(y - 2*z) - sin(y - z)) + 1.25663706143592*sin(y - 2*z) + 0.628318530717959*sin(y - z))/sqrt((0.01*x + 0.045)^2 + 1.0) + 0.628318530717959*(6.28318530717959*(0.01*x + 0.045)*(2*sin(y - 2*z) + sin(y - z)) - 0.628318530717959*sin(y - 2*z) - 0.628318530717959*sin(y - z))/sqrt((0.01*x + 0.045)^2 + 1.0))/sqrt((0.01*x + 0.045)^2 + 1.0) +div_par_solution = (0.01*x + 0.045)*(-12.5663706143592*cos(y - 2*z) - 6.28318530717959*cos(y - z) + 0.628318530717959*(cos(y - 2*z) + cos(y - z))/(0.01*x + 0.045))/sqrt((0.01*x + 0.045)^2 + 1.0) +div_par_K_grad_par_solution = (0.01*x + 0.045)*(6.28318530717959*sin(y - z) - 0.628318530717959*sin(y - z)/(0.01*x + 0.045))*(6.28318530717959*(0.01*x + 0.045)*(-2*cos(y - 2*z) - cos(y - z)) + 0.628318530717959*cos(y - 2*z) + 0.628318530717959*cos(y - z))/((0.01*x + 0.045)^2 + 1.0) + (6.28318530717959*(0.01*x + 0.045)*(6.28318530717959*(0.01*x + 0.045)*(-4*sin(y - 2*z) - sin(y - z)) + 1.25663706143592*sin(y - 2*z) + 0.628318530717959*sin(y - z))/sqrt((0.01*x + 0.045)^2 + 1.0) + 0.628318530717959*(6.28318530717959*(0.01*x + 0.045)*(2*sin(y - 2*z) + sin(y - z)) - 0.628318530717959*sin(y - 2*z) - 0.628318530717959*sin(y - z))/sqrt((0.01*x + 0.045)^2 + 1.0))*cos(y - z)/sqrt((0.01*x + 0.045)^2 + 1.0) +K = cos(y - z) +laplace_par_solution = (0.01*x + 0.045)*(6.28318530717959*(6.28318530717959*(0.01*x + 0.045)*(-4*sin(y - 2*z) - sin(y - z)) + 1.25663706143592*sin(y - 2*z) + 0.628318530717959*sin(y - z))/sqrt((0.01*x + 0.045)^2 + 1.0) + 0.628318530717959*(6.28318530717959*(0.01*x + 0.045)*(2*sin(y - 2*z) + sin(y - z)) - 0.628318530717959*sin(y - 2*z) - 0.628318530717959*sin(y - z))/((0.01*x + 0.045)*sqrt((0.01*x + 0.045)^2 + 1.0)))/sqrt((0.01*x + 0.045)^2 + 1.0) [mesh] symmetricglobalx = true file = fci.grid.nc +MXG = 1 +MYG = 1 +NXPE = 1 [mesh:ddy] first = C2 diff --git a/tests/MMS/spatial/fci/fci_mms.cxx b/tests/MMS/spatial/fci/fci_mms.cxx index 18405a7f88..7967452f3d 100644 --- a/tests/MMS/spatial/fci/fci_mms.cxx +++ b/tests/MMS/spatial/fci/fci_mms.cxx @@ -1,6 +1,41 @@ #include "bout/bout.hxx" -#include "bout/derivs.hxx" +#include "bout/build_config.hxx" +#include "bout/difops.hxx" +#include "bout/field.hxx" +#include "bout/field3d.hxx" #include "bout/field_factory.hxx" +#include "bout/globals.hxx" +#include "bout/options.hxx" +#include "bout/options_io.hxx" +#include "bout/utils.hxx" + +#include + +#include +#include + +namespace { +auto fci_op_test(const std::string& name, Options& dump, const Field3D& input, + const Field3D& result) { + auto* mesh = input.getMesh(); + const Field3D solution{FieldFactory::get()->create3D(fmt::format("{}_solution", name), + Options::getRoot(), mesh)}; + const Field3D error{result - solution}; + + dump[fmt::format("{}_l_2", name)] = sqrt(mean(SQ(error), true, "RGN_NOBNDRY")); + dump[fmt::format("{}_l_inf", name)] = max(abs(error), true, "RGN_NOBNDRY"); + + dump[fmt::format("{}_result", name)] = result; + dump[fmt::format("{}_error", name)] = error; + dump[fmt::format("{}_input", name)] = input; + dump[fmt::format("{}_solution", name)] = solution; + + for (int slice = 1; slice < mesh->ystart; ++slice) { + dump[fmt::format("{}_input.ynext(-{})", name, slice)] = input.ynext(-slice); + dump[fmt::format("{}_input.ynext({})", name, slice)] = input.ynext(slice); + } +} +} // namespace int main(int argc, char** argv) { BoutInitialise(argc, argv); @@ -8,30 +43,26 @@ int main(int argc, char** argv) { using bout::globals::mesh; Field3D input{FieldFactory::get()->create3D("input_field", Options::getRoot(), mesh)}; - Field3D solution{FieldFactory::get()->create3D("solution", Options::getRoot(), mesh)}; - - // Communicate to calculate parallel transform - mesh->communicate(input); + Field3D K{FieldFactory::get()->create3D("K", Options::getRoot(), mesh)}; - Field3D result{Grad_par(input)}; - Field3D error{result - solution}; + // Communicate to calculate parallel transform. + if constexpr (bout::build::use_metric_3d) { + // Div_par operators require B parallel slices: + // Coordinates::geometry doesn't ensure this (yet) + auto& Bxy = mesh->getCoordinates()->Bxy; + mesh->communicate(Bxy); + } + mesh->communicate(input, K); Options dump; // Add mesh geometry variables mesh->outputVars(dump); - dump["l_2"] = sqrt(mean(SQ(error), true, "RGN_NOBNDRY")); - dump["l_inf"] = max(abs(error), true, "RGN_NOBNDRY"); - - dump["result"] = result; - dump["error"] = error; - dump["input"] = input; - dump["solution"] = solution; - - for (int slice = 1; slice < mesh->ystart; ++slice) { - dump[fmt::format("input.ynext(-{})", slice)] = input.ynext(-slice); - dump[fmt::format("input.ynext({})", slice)] = input.ynext(slice); - } + fci_op_test("grad_par", dump, input, Grad_par(input)); + fci_op_test("grad2_par2", dump, input, Grad2_par2(input)); + fci_op_test("div_par", dump, input, Div_par(input)); + fci_op_test("div_par_K_grad_par", dump, input, Div_par_K_Grad_par(K, input)); + fci_op_test("laplace_par", dump, input, Laplace_par(input)); bout::writeDefaultOutputFile(dump); diff --git a/tests/MMS/spatial/fci/mms.py b/tests/MMS/spatial/fci/mms.py index 1e71135c90..b28e337ac0 100755 --- a/tests/MMS/spatial/fci/mms.py +++ b/tests/MMS/spatial/fci/mms.py @@ -3,13 +3,18 @@ # Generate manufactured solution and sources for FCI test # -from boutdata.mms import * +from math import pi +import warnings -from sympy import sin, cos, sqrt +from boututils.boutwarnings import AlwaysWarning +from boutdata.data import BoutOptionsFile +from boutdata.mms import diff, exprToStr, x, y, z +from sympy import sin, cos, sqrt, Expr -from math import pi +warnings.simplefilter("ignore", AlwaysWarning) f = sin(y - z) + sin(y - 2 * z) +K = cos(z - y) Lx = 0.1 Ly = 10.0 @@ -23,12 +28,42 @@ B = sqrt(Bpx**2 + Bt**2) -def FCI_ddy(f): +def FCI_grad_par(f: Expr) -> Expr: return (Bt * diff(f, y) * 2.0 * pi / Ly + Bpx * diff(f, z) * 2.0 * pi / Lz) / B +def FCI_grad2_par2(f: Expr) -> Expr: + return FCI_grad_par(FCI_grad_par(f)) + + +def FCI_div_par(f: Expr) -> Expr: + return Bpx * FCI_grad_par(f / Bpx) + + +def FCI_div_par_K_grad_par(f: Expr, K: Expr) -> Expr: + return (K * FCI_grad2_par2(f)) + (FCI_div_par(K) * FCI_grad_par(f)) + + +def FCI_Laplace_par(f: Expr) -> Expr: + return FCI_div_par(FCI_grad_par(f)) + + ############################################ # Equations solved -print("input = " + exprToStr(f)) -print("solution = " + exprToStr(FCI_ddy(f))) +options = BoutOptionsFile("data/BOUT.inp") + +for name, expr in ( + ("input_field", f), + ("K", K), + ("grad_par_solution", FCI_grad_par(f)), + ("grad2_par2_solution", FCI_grad2_par2(f)), + ("div_par_solution", FCI_div_par(f)), + ("div_par_K_grad_par_solution", FCI_div_par_K_grad_par(f, K)), + ("laplace_par_solution", FCI_Laplace_par(f)), +): + expr_str = exprToStr(expr) + print(f"{name} = {expr_str}") + options[name] = expr_str + +options.write("data/BOUT.inp", overwrite=True) diff --git a/tests/MMS/spatial/fci/runtest b/tests/MMS/spatial/fci/runtest index 7a9d6e655e..34340e53f4 100755 --- a/tests/MMS/spatial/fci/runtest +++ b/tests/MMS/spatial/fci/runtest @@ -6,209 +6,266 @@ # Cores: 2 # requires: zoidberg -from boututils.run_wrapper import build_and_log, launch_safe -from boutdata.collect import collect -import boutconfig as conf - -from numpy import array, log, polyfit, linspace, arange - -import pickle - -from sys import stdout +import argparse +import json +import pathlib +import sys +from time import time +import boutconfig as conf import zoidberg as zb - -nx = 4 # Not changed for these tests - +from boutdata.collect import collect +from boututils.run_wrapper import build_and_log, launch_safe +from numpy import arange, array, linspace, log, polyfit +from scipy.interpolate import RectBivariateSpline as RBS + +# Global parameters +DIRECTORY = "data" +NPROC = 2 +MTHREAD = 2 +OPERATORS = ("grad_par", "grad2_par2", "div_par", "div_par_K_grad_par", "laplace_par") +# Note that we need at least _2_ interior points for hermite spline +# interpolation due to an awkwardness with the boundaries +NX = 4 # Resolution in y and z -nlist = [8, 16, 32, 64, 128] - -# Number of parallel slices (in each direction) -nslices = [1] - -directory = "data" - -nproc = 2 -mthread = 2 - - -success = True - -error_2 = {} -error_inf = {} -method_orders = {} - -# Run with periodic Y? -yperiodic = True - -failures = [] - -build_and_log("FCI MMS test") - -for nslice in nslices: - for method in [ - "hermitespline", - "lagrange4pt", - "bilinear", - # "monotonichermitespline", - ]: - error_2[nslice] = [] - error_inf[nslice] = [] - - # Which central difference scheme to use and its expected order - order = nslice * 2 - method_orders[nslice] = {"name": "C{}".format(order), "order": order} - - for n in nlist: - # Define the magnetic field using new poloidal gridding method - # Note that the Bz and Bzprime parameters here must be the same as in mms.py - field = zb.field.Slab(Bz=0.05, Bzprime=0.1) - # Create rectangular poloidal grids - poloidal_grid = zb.poloidal_grid.RectangularPoloidalGrid( - nx, n, 0.1, 1.0, MXG=1 - ) - # Set the ylength and y locations - ylength = 10.0 - - if yperiodic: - ycoords = linspace(0.0, ylength, n, endpoint=False) - else: - # Doesn't include the end points - ycoords = (arange(n) + 0.5) * ylength / float(n) - - # Create the grid - grid = zb.grid.Grid(poloidal_grid, ycoords, ylength, yperiodic=yperiodic) - # Make and write maps - maps = zb.make_maps(grid, field, nslice=nslice, quiet=True, MXG=1) - zb.write_maps( - grid, - field, - maps, - new_names=False, - metric2d=conf.isMetric2D(), - quiet=True, - ) - - args = " MZ={} MYG={} mesh:paralleltransform:y_periodic={} mesh:ddy:first={} NXPE={}".format( - n, - nslice, - yperiodic, - method_orders[nslice]["name"], - 2 if conf.has["petsc"] and method == "hermitespline" else 1, - ) - args += f" mesh:paralleltransform:xzinterpolation:type={method}" - - # Command to run - cmd = "./fci_mms " + args - - print("Running command: " + cmd) - - # Launch using MPI - s, out = launch_safe(cmd, nproc=nproc, mthread=mthread, pipe=True) - - # Save output to log file - with open("run.log." + str(n), "w") as f: - f.write(out) - - if s: - print("Run failed!\nOutput was:\n") - print(out) - exit(s) - - # Collect data - l_2 = collect( - "l_2", - tind=[1, 1], - info=False, - path=directory, - xguards=False, - yguards=False, - ) - l_inf = collect( - "l_inf", - tind=[1, 1], - info=False, - path=directory, - xguards=False, - yguards=False, - ) - - error_2[nslice].append(l_2) - error_inf[nslice].append(l_inf) - - print("Errors : l-2 {:f} l-inf {:f}".format(l_2, l_inf)) - - dx = 1.0 / array(nlist) - - # Calculate convergence order - fit = polyfit(log(dx), log(error_2[nslice]), 1) - order = fit[0] - stdout.write("Convergence order = {:f} (fit)".format(order)) - - order = log(error_2[nslice][-2] / error_2[nslice][-1]) / log(dx[-2] / dx[-1]) - stdout.write(", {:f} (small spacing)".format(order)) - - # Should be close to the expected order - if order > method_orders[nslice]["order"] * 0.95: - print("............ PASS\n") - else: - print("............ FAIL\n") - success = False - failures.append(method_orders[nslice]["name"]) - - -with open("fci_mms.pkl", "wb") as output: - pickle.dump(nlist, output) - for nslice in nslices: - pickle.dump(error_2[nslice], output) - pickle.dump(error_inf[nslice], output) - -# Do we want to show the plot as well as save it to file. -showPlot = True - -if False: +NLIST = [8, 16, 32, 64] +dx = 1.0 / array(NLIST) + + +def myRBS(a, b, c): + """RectBivariateSpline, but automatically tune spline degree for small arrays""" + mx, _ = c.shape + kx = max(mx - 1, 1) + kx = min(kx, 3) + return RBS(a, b, c, kx=kx) + + +zb.poloidal_grid.RectBivariateSpline = myRBS + + +def quiet_collect(name: str) -> float: + # Index to return a plain (numpy) float rather than `BoutArray` + return collect( + name, + tind=[1, 1], + info=False, + path=DIRECTORY, + xguards=False, + yguards=False, + )[()] + + +def assert_convergence(error, dx, name, expected) -> bool: + fit = polyfit(log(dx), log(error), 1) + order = fit[0] + print(f"{name} convergence order = {order:f} (fit)", end="") + + order = log(error[-2] / error[-1]) / log(dx[-2] / dx[-1]) + print(f", {order:f} (small spacing)", end="") + + # Should be close to the expected order + success = order > expected * 0.95 + print(f"\t............ {'PASS' if success else 'FAIL'}") + + return success + + +def run_fci_operators( + nslice: int, nz: int, yperiodic: bool, name: str +) -> dict[str, float]: + # Define the magnetic field using new poloidal gridding method + # Note that the Bz and Bzprime parameters here must be the same as in mms.py + field = zb.field.Slab(Bz=0.05, Bzprime=0.1) + # Create rectangular poloidal grids + poloidal_grid = zb.poloidal_grid.RectangularPoloidalGrid(NX, nz, 0.1, 1.0, MXG=1) + # Set the ylength and y locations + ylength = 10.0 + + if yperiodic: + ycoords = linspace(0.0, ylength, nz, endpoint=False) + else: + # Doesn't include the end points + ycoords = (arange(nz) + 0.5) * ylength / float(nz) + + # Create the grid + grid = zb.grid.Grid(poloidal_grid, ycoords, ylength, yperiodic=yperiodic) + maps = zb.make_maps(grid, field, nslice=nslice, quiet=True, MXG=1) + zb.write_maps( + grid, + field, + maps, + new_names=False, + metric2d=conf.isMetric2D(), + quiet=True, + ) + + # Command to run + args = f"MZ={nz} MYG={nslice} mesh:paralleltransform:y_periodic={yperiodic} {name}" + cmd = f"./fci_mms {args}" + print(f"Running command: {cmd}", end="") + + # Launch using MPI + start = time() + status, out = launch_safe(cmd, nproc=NPROC, mthread=MTHREAD, pipe=True) + print(f" ... done in {time() - start:.3}s") + + # Save output to log file + pathlib.Path(f"run.log.{nz}").write_text(out) + + if status: + print(f"Run failed!\nOutput was:\n{out}") + sys.exit(status) + + return { + operator: { + "l_2": quiet_collect(f"{operator}_l_2"), + "l_inf": quiet_collect(f"{operator}_l_inf"), + } + for operator in OPERATORS + } + + +def transpose( + errors: list[dict[str, dict[str, float]]], +) -> dict[str, dict[str, list[float]]]: + """Turn a list of {operator: error} into a dict of {operator: [errors]}""" + + kinds = ("l_2", "l_inf") + result = {operator: {kind: [] for kind in kinds} for operator in OPERATORS} + for error in errors: + for k, v in error.items(): + for kind in kinds: + result[k][kind].append(v[kind]) + return result + + +def check_fci_operators(name: str, case: dict) -> bool: + failures = [] + + nslice = case["nslice"] + yperiodic = case["yperiodic"] + order = case["order"] + args = case["args"] + + all_errors = [] + + for n in NLIST: + errors = run_fci_operators(nslice, n, yperiodic, args) + all_errors.append(errors) + + for operator in OPERATORS: + l_2 = errors[operator]["l_2"] + l_inf = errors[operator]["l_inf"] + + print(f"{operator} errors: l-2 {l_2:f} l-inf {l_inf:f}") + + final_errors = transpose(all_errors) + for operator in OPERATORS: + test_name = f"{operator} {name}" + success = assert_convergence( + final_errors[operator]["l_2"], dx, test_name, order + ) + if not success: + failures.append(test_name) + + return final_errors, failures + + +def make_plots(cases: dict[str, dict]): try: - # Plot using matplotlib if available import matplotlib.pyplot as plt + except ImportError: + print("No matplotlib") + return - fig, ax = plt.subplots(1, 1) - - for nslice in nslices: - ax.plot( - dx, - error_2[nslice], - "-", - label="{} $l_2$".format(method_orders[nslice]["name"]), - ) - ax.plot( - dx, - error_inf[nslice], - "--", - label="{} $l_\\inf$".format(method_orders[nslice]["name"]), - ) + num_operators = len(OPERATORS) + fig, axes = plt.subplots(1, num_operators, figsize=(num_operators * 4, 4)) + + for ax, operator in zip(axes, OPERATORS): + for name, case in cases.items(): + ax.loglog(dx, case[operator]["l_2"], "-", label=f"{name} $l_2$") + ax.loglog(dx, case[operator]["l_inf"], "--", label=f"{name} $l_\\inf$") ax.legend(loc="upper left") ax.grid() - ax.set_yscale("log") - ax.set_xscale("log") - ax.set_title("error scaling") + ax.set_title(f"Error scaling for {operator}") ax.set_xlabel(r"Mesh spacing $\delta x$") ax.set_ylabel("Error norm") - plt.savefig("fci_mms.pdf") - - print("Plot saved to fci_mms.pdf") - - if showPlot: - plt.show() - plt.close() - except ImportError: - print("No matplotlib") - -if success: - print("All tests passed") - exit(0) -else: - print("Some tests failed:") - for failure in failures: - print("\t" + failure) - exit(1) + fig.tight_layout() + fig.savefig("fci_mms.pdf") + print("Plot saved to fci_mms.pdf") + + if args.show_plots: + plt.show() + plt.close() + + +if __name__ == "__main__": + build_and_log("FCI MMS test") + + parser = argparse.ArgumentParser("Error scaling test for FCI operators") + parser.add_argument( + "--make-plots", action="store_true", help="Create plots of error scaling" + ) + parser.add_argument( + "--show-plots", + action="store_true", + help="Stop and show plots, implies --make-plots", + ) + parser.add_argument( + "--dump-errors", + type=str, + help="Output file to dump errors as JSON", + default="fci_operator_errors.json", + ) + + args = parser.parse_args() + + success = True + failures = [] + cases = { + "nslice=1 hermitespline": { + "nslice": 1, + "order": 2, + "yperiodic": True, + "args": "mesh:ddy:first=C2 mesh:paralleltransform:xzinterpolation:type=hermitespline", + }, + "nslice=1 lagrange4pt": { + "nslice": 1, + "order": 2, + "yperiodic": True, + "args": "mesh:ddy:first=C2 mesh:paralleltransform:xzinterpolation:type=lagrange4pt", + }, + "nslice=1 monotonichermitespline": { + "nslice": 1, + "order": 2, + "yperiodic": True, + "args": ( + "mesh:ddy:first=C2 " + "mesh:paralleltransform:xzinterpolation:type=monotonichermitespline " + "mesh:paralleltransform:xzinterpolation:rtol=1e-3 " + "mesh:paralleltransform:xzinterpolation:atol=5e-3" + ), + }, + } + + for name, case in cases.items(): + error2, failures_ = check_fci_operators(name, case) + case.update(error2) + failures.extend(failures_) + success &= len(failures) == 0 + + if args.dump_errors: + pathlib.Path(args.dump_errors).write_text(json.dumps(cases)) + + if args.make_plots or args.show_plots: + make_plots(cases) + + if success: + print("\nAll tests passed") + else: + print("\nSome tests failed:") + for failure in failures: + print(f"\t{failure}") + + sys.exit(0 if success else 1) diff --git a/tests/integrated/test-fci-boundary/get_par_bndry.cxx b/tests/integrated/test-fci-boundary/get_par_bndry.cxx index ac0f5de2a6..8e3cfac2f7 100644 --- a/tests/integrated/test-fci-boundary/get_par_bndry.cxx +++ b/tests/integrated/test-fci-boundary/get_par_bndry.cxx @@ -1,31 +1,39 @@ #include "bout/bout.hxx" -#include "bout/derivs.hxx" +#include "bout/field3d.hxx" #include "bout/field_factory.hxx" +#include "bout/globals.hxx" +#include "bout/options.hxx" +#include "bout/options_io.hxx" +#include "bout/output.hxx" #include "bout/parallel_boundary_region.hxx" +#include + +#include + int main(int argc, char** argv) { BoutInitialise(argc, argv); using bout::globals::mesh; - std::vector fields; - fields.resize(static_cast(BoundaryParType::SIZE)); + std::vector fields(static_cast(BoundaryParType::SIZE), Field3D{0.0}); + Options dump; for (int i = 0; i < fields.size(); i++) { - fields[i] = Field3D{0.0}; + fields[i].allocate(); + const auto boundary = static_cast(i); + const auto boundary_name = toString(boundary); mesh->communicate(fields[i]); - for (const auto& bndry_par : - mesh->getBoundariesPar(static_cast(i))) { - output.write("{:s} region\n", toString(static_cast(i))); + for (const auto& bndry_par : mesh->getBoundariesPar(boundary)) { + output.write("{:s} region\n", boundary_name); for (bndry_par->first(); !bndry_par->isDone(); bndry_par->next()) { fields[i][bndry_par->ind()] += 1; - output.write("{:s} increment\n", toString(static_cast(i))); + output.write("{:s} increment\n", boundary_name); } } - output.write("{:s} done\n", toString(static_cast(i))); + output.write("{:s} done\n", boundary_name); - dump[fmt::format("field_{:s}", toString(static_cast(i)))] = - fields[i]; + dump[fmt::format("field_{:s}", boundary_name)] = fields[i]; } bout::writeDefaultOutputFile(dump); diff --git a/tests/integrated/test-fci-boundary/runtest b/tests/integrated/test-fci-boundary/runtest index 1b1460da53..e749055185 100755 --- a/tests/integrated/test-fci-boundary/runtest +++ b/tests/integrated/test-fci-boundary/runtest @@ -1,29 +1,15 @@ #!/usr/bin/env python3 # # Python script to run and analyse MMS test -# -# Cores: 2 -# only working with cmake -# requires: False from boututils.run_wrapper import launch_safe from boututils.datafile import DataFile -from boutdata.collect import collect as _collect +from boutdata.collect import collect import numpy as np -def collect(var): - return _collect( - var, - info=False, - path=directory, - xguards=False, - yguards=False, - ) - - -nprocs = [1] # , 2, 4] +nprocs = [1] mthread = 2 directory = "data" @@ -43,11 +29,6 @@ regions = { } regions = {k: v.astype(int) for k, v in regions.items()} -# for x in "xout", "xin": -# regions[x] = np.logical_or(regions[f"{x}_fwd"], regions[f"{x}_bwd"]) -# for x in "fwd", "bwd": -# regions[x] = np.logical_or(regions[f"xin_{x}"], regions[f"xout_{x}"]) -# regions["all"] = np.logical_or(regions["xin"], regions["xout"]) for x in "xout", "xin": regions[x] = regions[f"{x}_fwd"] + regions[f"{x}_bwd"] for x in "fwd", "bwd": @@ -56,15 +37,18 @@ regions["all"] = regions["xin"] + regions["xout"] for nproc in nprocs: cmd = "./get_par_bndry" - - # Launch using MPI _, out = launch_safe(cmd, nproc=nproc, mthread=mthread, pipe=True) for k, v in regions.items(): - # Collect data - data = collect(f"field_{k}") + data = collect( + f"field_{k}", + info=False, + path=directory, + xguards=False, + yguards=False, + ) assert np.allclose(data, v), ( - k + " does not match", + f"{k} does not match", np.sum(data), np.sum(v), np.max(data), diff --git a/tests/integrated/test-fci-mpi/fci_mpi.cxx b/tests/integrated/test-fci-mpi/fci_mpi.cxx index 94520dd4a6..94e8e878ef 100644 --- a/tests/integrated/test-fci-mpi/fci_mpi.cxx +++ b/tests/integrated/test-fci-mpi/fci_mpi.cxx @@ -1,38 +1,41 @@ +#include "fmt/format.h" #include "bout/bout.hxx" -#include "bout/derivs.hxx" +#include "bout/field3d.hxx" #include "bout/field_factory.hxx" +#include "bout/globals.hxx" +#include "bout/options.hxx" +#include "bout/options_io.hxx" +#include "bout/region.hxx" + +namespace { +auto fci_mpi_test(int num, Options& dump) { + using bout::globals::mesh; + Field3D input{FieldFactory::get()->create3D(fmt::format("input_{:d}:function", num), + Options::getRoot(), mesh)}; + mesh->communicate(input); + input.applyParallelBoundary("parallel_neumann_o2"); + + for (int slice = -mesh->ystart; slice <= mesh->ystart; ++slice) { + if (slice == 0) { + continue; + } + Field3D tmp{0.}; + BOUT_FOR(i, tmp.getRegion("RGN_NOBNDRY")) { + tmp[i] = input.ynext(slice)[i.yp(slice)]; + } + dump[fmt::format("output_{:d}_{:+d}", num, slice)] = tmp; + } +} +} // namespace int main(int argc, char** argv) { BoutInitialise(argc, argv); - { - using bout::globals::mesh; - Options* options = Options::getRoot(); - int i = 0; - const std::string default_str{"not_set"}; - Options dump; - while (true) { - std::string temp_str; - options->get(fmt::format("input_{:d}:function", i), temp_str, default_str); - if (temp_str == default_str) { - break; - } - Field3D input{FieldFactory::get()->create3D(fmt::format("input_{:d}:function", i), - Options::getRoot(), mesh)}; - // options->get(fmt::format("input_{:d}:boundary_perp", i), temp_str, s"free_o3"); - mesh->communicate(input); - input.applyParallelBoundary("parallel_neumann_o2"); - for (int slice = -mesh->ystart; slice <= mesh->ystart; ++slice) { - if (slice != 0) { - Field3D tmp{0.}; - BOUT_FOR(i, tmp.getRegion("RGN_NOBNDRY")) { - tmp[i] = input.ynext(slice)[i.yp(slice)]; - } - dump[fmt::format("output_{:d}_{:+d}", i, slice)] = tmp; - } - } - ++i; - } - bout::writeDefaultOutputFile(dump); + Options dump; + + for (auto num : {0, 1, 2, 3}) { + fci_mpi_test(num, dump); } + + bout::writeDefaultOutputFile(dump); BoutFinalise(); } diff --git a/tests/integrated/test-fci-mpi/runtest b/tests/integrated/test-fci-mpi/runtest index 6676f8f7a5..c18ab0391d 100755 --- a/tests/integrated/test-fci-mpi/runtest +++ b/tests/integrated/test-fci-mpi/runtest @@ -1,57 +1,82 @@ #!/usr/bin/env python3 # # Python script to run and analyse MMS test -# - -# Cores: 8 -# requires: metric_3d -from boututils.run_wrapper import build_and_log, launch_safe, shell_safe +from boututils.run_wrapper import build_and_log, launch_safe from boutdata.collect import collect -import boutconfig as conf import itertools +import sys -import numpy as np +import numpy.testing as npt # Resolution in x and y -nlist = [1, 2, 4] +NLIST = [1, 2, 4] +MAXCORES = 8 +NSLICES = [1] -maxcores = 8 +build_and_log("FCI MMS test") -nslices = [1] +COLLECT_KW = dict(info=False, xguards=False, yguards=False, path="data") -success = True -build_and_log("FCI MMS test") +def run_case(nxpe: int, nype: int, mthread: int): + cmd = f"./fci_mpi NXPE={nxpe} NYPE={nype}" + print(f"Running command: {cmd}") + + _, out = launch_safe(cmd, nproc=nxpe * nype, mthread=mthread, pipe=True) + + # Save output to log file + with open(f"run.log.{nxpe}.{nype}.{nslice}.log", "w") as f: + f.write(out) + + +def test_case(nxpe: int, nype: int, mthread: int, ref: dict) -> bool: + run_case(nxpe, nype, mthread) + + failures = [] + + for name, val in ref.items(): + try: + npt.assert_allclose(val, collect(name, **COLLECT_KW)) + except AssertionError as e: + failures.append((nxpe, nype, name, e)) -for nslice in nslices: - for NXPE, NYPE in itertools.product(nlist, nlist): - if NXPE * NYPE > maxcores: + return failures + + +failures = [] + +for nslice in NSLICES: + # reference data! + run_case(1, 1, MAXCORES) + + ref = {} + for i in range(4): + for yp in range(1, nslice + 1): + for y in [-yp, yp]: + name = f"output_{i}_{y:+d}" + ref[name] = collect(name, **COLLECT_KW) + + for nxpe, nype in itertools.product(NLIST, NLIST): + if (nxpe, nype) == (1, 1): + # reference case, done above continue - args = f"NXPE={NXPE} NYPE={NYPE}" - # Command to run - cmd = f"./fci_mpi {args}" - - print(f"Running command: {cmd}") - - mthread = maxcores // (NXPE * NYPE) - # Launch using MPI - _, out = launch_safe(cmd, nproc=NXPE * NYPE, mthread=mthread, pipe=True) - - # Save output to log file - with open(f"run.log.{NXPE}.{NYPE}.{nslice}.log", "w") as f: - f.write(out) - - collect_kw = dict(info=False, xguards=False, yguards=False, path="data") - if NXPE == NYPE == 1: - # reference data! - ref = {} - for i in range(4): - for yp in range(1, nslice + 1): - for y in [-yp, yp]: - name = f"output_{i}_{y:+d}" - ref[name] = collect(name, **collect_kw) - else: - for name, val in ref.items(): - assert np.allclose(val, collect(name, **collect_kw)) + if nxpe * nype > MAXCORES: + continue + + mthread = MAXCORES // (nxpe * nype) + failures_ = test_case(nxpe, nype, mthread, ref) + failures.extend(failures_) + + +success = len(failures) == 0 +if success: + print("\nAll tests passed") +else: + print("\nSome tests failed:") + for nxpe, nype, name, error in failures: + print("----------") + print(f"case {nxpe=} {nype=} {name=}\n{error}") + +sys.exit(0 if success else 1)