diff --git a/core/unit/ctest_dg_lowpass_filter.c b/core/unit/ctest_dg_lowpass_filter.c new file mode 100644 index 0000000000..ac951f7efe --- /dev/null +++ b/core/unit/ctest_dg_lowpass_filter.c @@ -0,0 +1,342 @@ +// Test the dg_lowpass_filter updater. +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#define FILTER_DIR 0 // Direction filtered in every test below. +#define FOUT_FILL 7.0 // Value fout is cleared to before each application. + +static struct gkyl_array* +mkarr(bool on_gpu, long nc, long size) +{ + struct gkyl_array* a = on_gpu? gkyl_array_cu_dev_new(GKYL_DOUBLE, nc, size) + : gkyl_array_new(GKYL_DOUBLE, nc, size); + return a; +} + +static double +filter_gain(int M, double fc, double freq) +{ + // Frequency response of the kernel at freq cycles/cell. + double wsum = 0.0, gain = 0.0; + for (int k=-M; kmode_num*xn[FILTER_DIR])*transverse_mod(xn, pctx->ndim); +} + +void eval_bump(double t, const double *xn, double *fout, void *ctx) +{ + // Off-center bump along the filtered direction. + struct profile_ctx *pctx = ctx; + fout[0] = (0.1 + exp(-pow((xn[FILTER_DIR]-pctx->x0)/pctx->w, 2)))*transverse_mod(xn, pctx->ndim); +} + +struct filter_env { + bool use_gpu; + struct gkyl_rect_grid grid; + struct gkyl_basis basis; + struct gkyl_range local, local_ext; + struct gkyl_array *fin, *fout; // What the updater sees. + struct gkyl_array *fin_ho, *fout_ho; // Host copies the checks read. +}; + +static void +filter_env_new(struct filter_env *env, bool use_gpu, int ndim, const int *cells, int poly_order) +{ + // Grid, basis, ranges and arrays for one test, on the unit cube. + double lower[GKYL_MAX_DIM], upper[GKYL_MAX_DIM]; + int nghost[GKYL_MAX_DIM]; + for (int d=0; duse_gpu = use_gpu; + gkyl_rect_grid_init(&env->grid, ndim, lower, upper, cells); + gkyl_cart_modal_serendip(&env->basis, ndim, poly_order); + gkyl_create_grid_ranges(&env->grid, nghost, &env->local_ext, &env->local); + + env->fin = mkarr(use_gpu, env->basis.num_basis, env->local_ext.volume); + env->fout = mkarr(use_gpu, env->basis.num_basis, env->local_ext.volume); + env->fin_ho = use_gpu? mkarr(false, env->fin->ncomp, env->fin->size) : gkyl_array_acquire(env->fin); + env->fout_ho = use_gpu? mkarr(false, env->fout->ncomp, env->fout->size) : gkyl_array_acquire(env->fout); +} + +static void +filter_env_release(struct filter_env *env) +{ + gkyl_array_release(env->fin); + gkyl_array_release(env->fout); + gkyl_array_release(env->fin_ho); + gkyl_array_release(env->fout_ho); +} + +static void +filter_apply(struct filter_env *env, const struct gkyl_range *sub, int M, + double cutoff_wavelength, evalf_t func, void *func_ctx) +{ + // Project func and filter it over sub, leaving both fields on the host. + gkyl_proj_on_basis *proj = gkyl_proj_on_basis_new(&env->grid, &env->basis, + env->basis.poly_order+1, 1, func, func_ctx); + gkyl_proj_on_basis_advance(proj, 0.0, &env->local, env->fin_ho); + gkyl_proj_on_basis_release(proj); + gkyl_array_copy(env->fin, env->fin_ho); + + gkyl_array_clear(env->fout, FOUT_FILL); + struct gkyl_dg_lowpass_filter *lpf = gkyl_dg_lowpass_filter_new(FILTER_DIR, M, + cutoff_wavelength, &env->basis, &env->grid, sub, env->use_gpu); + gkyl_dg_lowpass_filter_advance(lpf, env->fin, env->fout); + gkyl_dg_lowpass_filter_release(lpf); + gkyl_array_copy(env->fout_ho, env->fout); +} + +static double +filter_integral_check(struct filter_env *env, const struct gkyl_range *sub, + double tol, const char *what) +{ + // Change of the integral over sub, as a fraction of the mass in the range. + double tot_in = 0.0, tot_out = 0.0, mass = 0.0; + struct gkyl_range_iter iter; + gkyl_range_iter_init(&iter, sub); + while (gkyl_range_iter_next(&iter)) { + long linidx = gkyl_range_idx(sub, iter.idx); + double in = ((const double *) gkyl_array_cfetch(env->fin_ho, linidx))[0]; + tot_in += in; + mass += fabs(in); + tot_out += ((const double *) gkyl_array_cfetch(env->fout_ho, linidx))[0]; + } + + double rel = (tot_out-tot_in)/mass; + TEST_CHECK( fabs(rel) < tol ); + TEST_MSG("%s: the integral changed by %.3e of the mass in the range", what, rel); + return rel; +} + +static void +filter_gain_check(struct filter_env *env, const struct gkyl_range *sub, int edge_skip, + double gain, double tol, const char *what) +{ + // Check fout = gain*fin, skipping edge_skip cells at each edge of sub. + struct gkyl_range_iter iter; + gkyl_range_iter_init(&iter, sub); + while (gkyl_range_iter_next(&iter)) { + if (iter.idx[FILTER_DIR] < sub->lower[FILTER_DIR]+edge_skip || + iter.idx[FILTER_DIR] > sub->upper[FILTER_DIR]-edge_skip) + continue; + + long linidx = gkyl_range_idx(sub, iter.idx); + const double *in_c = gkyl_array_cfetch(env->fin_ho, linidx); + const double *out_c = gkyl_array_cfetch(env->fout_ho, linidx); + for (int c=0; cbasis.num_basis; c++) { + TEST_CHECK( fabs(out_c[c] - gain*in_c[c]) < tol ); + TEST_MSG("%s, cell %d coeff %d: expected %.13e, got %.13e", + what, iter.idx[FILTER_DIR], c, gain*in_c[c], out_c[c]); + } + } +} + +static void +test_response(bool use_gpu, int ndim, const int *cells) +{ + // Away from the edges the filter scales each field by the kernel response. + int M = 8; + double fc = 0.3; + + struct filter_env env; + filter_env_new(&env, use_gpu, ndim, cells, 1); + double cutoff = env.grid.dx[FILTER_DIR]/fc; + + double mod_max = 1.0; // Peak of the transverse modulation. + for (int d=0; d +#include +#include + +struct gkyl_dg_lowpass_filter* +gkyl_dg_lowpass_filter_new(int dir, int half_width, double cutoff_wavelength, + const struct gkyl_basis *basis, const struct gkyl_rect_grid *grid, + const struct gkyl_range *range, bool use_gpu) +{ + // Allocate space for new updater. + struct gkyl_dg_lowpass_filter *up = gkyl_malloc(sizeof(*up)); + + up->use_gpu = use_gpu; + up->ndim = basis->ndim; + up->dir = dir; + up->half_width = half_width; + up->num_basis = basis->num_basis; + up->grid = *grid; + up->range = *range; + + // Reflect the stencil only at faces that coincide with the domain boundary. + up->reflect_lo = range->lower[dir] == 1; + up->reflect_up = range->upper[dir] == grid->cells[dir]; + + // Perform some basic checks: + assert(grid->ndim == range->ndim); + assert(0 <= dir && dir < grid->ndim); + assert(half_width > 0); + + // Normalized cutoff frequency in cycles per cell. + double fc = grid->dx[dir]/cutoff_wavelength; + assert(0.0 < fc && fc <= 0.5); + + up->weights = gkyl_malloc((2*half_width+1)*sizeof(double)); + dg_lpf_calc_weights(half_width, fc, up->weights); + + // Sign each coefficient picks up when its donor cell is reached by mirroring. + up->sign_plain = gkyl_malloc(up->num_basis*sizeof(double)); + up->sign_mirror = gkyl_malloc(up->num_basis*sizeof(double)); + for (int c=0; cnum_basis; c++) up->sign_plain[c] = 1.0; + basis->flip_odd_sign(dir, up->sign_plain, up->sign_mirror); + + up->weights_cu = up->sign_plain_cu = up->sign_mirror_cu = NULL; +#ifdef GKYL_HAVE_CUDA + if (up->use_gpu) { + up->weights_cu = gkyl_cu_malloc((2*half_width+1)*sizeof(double)); + up->sign_plain_cu = gkyl_cu_malloc(up->num_basis*sizeof(double)); + up->sign_mirror_cu = gkyl_cu_malloc(up->num_basis*sizeof(double)); + gkyl_cu_memcpy(up->weights_cu, up->weights, (2*half_width+1)*sizeof(double), GKYL_CU_MEMCPY_H2D); + gkyl_cu_memcpy(up->sign_plain_cu, up->sign_plain, up->num_basis*sizeof(double), GKYL_CU_MEMCPY_H2D); + gkyl_cu_memcpy(up->sign_mirror_cu, up->sign_mirror, up->num_basis*sizeof(double), GKYL_CU_MEMCPY_H2D); + } +#endif + + return up; +} + +void +gkyl_dg_lowpass_filter_advance(gkyl_dg_lowpass_filter *up, + struct gkyl_array *GKYL_RESTRICT fdo, struct gkyl_array *GKYL_RESTRICT ftar) +{ + assert(fdo != ftar); // Stencil operation, can't be done in-place. + assert(fdo->ncomp == ftar->ncomp); + assert(fdo->size == ftar->size); + assert(fdo->ncomp == up->num_basis); + +#ifdef GKYL_HAVE_CUDA + if (up->use_gpu) { + gkyl_dg_lowpass_filter_advance_cu(up, fdo, ftar); + return; + } +#endif + + int dir = up->dir; + int M = up->half_width; + int num_basis = up->num_basis; + int idx_do[GKYL_MAX_DIM]; + + // Loop over the target range. + struct gkyl_range_iter iter; + gkyl_range_iter_init(&iter, &up->range); + while (gkyl_range_iter_next(&iter)) { + + long linidx_tar = gkyl_range_idx(&up->range, iter.idx); + double *ftar_c = gkyl_array_fetch(ftar, linidx_tar); + + gkyl_copy_int_arr(up->range.ndim, iter.idx, idx_do); + + for (int c=0; crange.lower[dir]) { + if (!up->reflect_lo) continue; // Interior edge: drop donor, renormalize. + idx_do[dir] = dg_lpf_mirror_idx(idx_k, up->range.lower[dir], up->range.upper[dir], &mirrored); + } + else if (idx_k > up->range.upper[dir]) { + if (!up->reflect_up) continue; // Interior edge: drop donor, renormalize. + idx_do[dir] = dg_lpf_mirror_idx(idx_k, up->range.lower[dir], up->range.upper[dir], &mirrored); + } + else { + idx_do[dir] = idx_k; + } + + long linidx_do = gkyl_range_idx(&up->range, idx_do); + const double *fdo_c = gkyl_array_cfetch(fdo, linidx_do); + + // Handle reflection. + const double *sgn = mirrored? up->sign_mirror : up->sign_plain; + + double w = up->weights[k+M]; + wsum += w; + for (int c=0; cweights); + gkyl_free(up->sign_plain); + gkyl_free(up->sign_mirror); +#ifdef GKYL_HAVE_CUDA + if (up->use_gpu) { + gkyl_cu_free(up->weights_cu); + gkyl_cu_free(up->sign_plain_cu); + gkyl_cu_free(up->sign_mirror_cu); + } +#endif + gkyl_free(up); +} diff --git a/core/zero/dg_lowpass_filter_cu.cu b/core/zero/dg_lowpass_filter_cu.cu new file mode 100644 index 0000000000..d54b65b341 --- /dev/null +++ b/core/zero/dg_lowpass_filter_cu.cu @@ -0,0 +1,77 @@ +/* -*- c++ -*- */ + +extern "C" { +#include +#include +#include +#include +} + +__global__ static void +gkyl_dg_lowpass_filter_advance_cu_ker(int dir, int M, int num_basis, + bool reflect_lo, bool reflect_up, + const double *GKYL_RESTRICT weights, const double *GKYL_RESTRICT sign_plain, + const double *GKYL_RESTRICT sign_mirror, struct gkyl_range range, + const struct gkyl_array *GKYL_RESTRICT fdo, struct gkyl_array *GKYL_RESTRICT ftar) +{ + int idx_tar[GKYL_MAX_DIM]; + int idx_do[GKYL_MAX_DIM]; + + for (unsigned long tid = threadIdx.x + blockIdx.x*blockDim.x; + tid < range.volume; tid += blockDim.x*gridDim.x) { + gkyl_sub_range_inv_idx(&range, tid, idx_tar); + + long linidx_tar = gkyl_range_idx(&range, idx_tar); + double *ftar_c = (double *) gkyl_array_fetch(ftar, linidx_tar); + + for (int c=0; c range.upper[dir]) { + if (!reflect_up) continue; // Interior edge: drop donor, renormalize. + idx_do[dir] = dg_lpf_mirror_idx(idx_k, range.lower[dir], range.upper[dir], &mirrored); + } + else { + idx_do[dir] = idx_k; + } + + long linidx_do = gkyl_range_idx(&range, idx_do); + const double *fdo_c = (const double *) gkyl_array_cfetch(fdo, linidx_do); + + const double *sgn = mirrored? sign_mirror : sign_plain; + + double w = weights[k+M]; + wsum += w; + for (int c=0; crange.nblocks, nthreads = up->range.nthreads; + + gkyl_dg_lowpass_filter_advance_cu_ker<<>> + (up->dir, up->half_width, up->num_basis, up->reflect_lo, up->reflect_up, + up->weights_cu, up->sign_plain_cu, up->sign_mirror_cu, up->range, + fdo->on_dev, ftar->on_dev); +} diff --git a/core/zero/gkyl_dg_lowpass_filter.h b/core/zero/gkyl_dg_lowpass_filter.h new file mode 100644 index 0000000000..8620d14d1b --- /dev/null +++ b/core/zero/gkyl_dg_lowpass_filter.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include +#include +#include + +// Object type. +typedef struct gkyl_dg_lowpass_filter gkyl_dg_lowpass_filter; + +/** + * Create a new updater that low-pass filters a DG field along one direction + * with a Blackman-windowed sinc kernel, normalized to preserve the zero mode. + * + * @param dir Direction along which to filter. + * @param half_width Stencil half-width M in cells (stencil spans 2M+1 cells). + * @param cutoff_wavelength Cutoff wavelength (physical units). Normalized + * cutoff f_c = dx/cutoff_wavelength must satisfy 0 < f_c <= 0.5. + * @param basis DG basis of the filtered field. + * @param grid Grid the filtered field is defined on. + * @param range Range to filter in. The stencil is reflected or truncated at the edges. + * @param use_gpu bool to determine if on GPU. + * @return New filter updater. + */ +struct gkyl_dg_lowpass_filter* +gkyl_dg_lowpass_filter_new(int dir, int half_width, double cutoff_wavelength, + const struct gkyl_basis *basis, const struct gkyl_rect_grid *grid, + const struct gkyl_range *range, bool use_gpu); + +/** + * Run the filter updater. Cannot be used in-place; ftar cells outside the + * range are left untouched. + * + * @param up Filter updater. + * @param fdo Donor field. + * @param ftar Target (filtered) field. + */ +void +gkyl_dg_lowpass_filter_advance(gkyl_dg_lowpass_filter *up, + struct gkyl_array *fdo, struct gkyl_array *ftar); + +/** + * Release the memory associated with this filter updater. + * + * @param up Filter updater. + */ +void +gkyl_dg_lowpass_filter_release(gkyl_dg_lowpass_filter *up); diff --git a/core/zero/gkyl_dg_lowpass_filter_priv.h b/core/zero/gkyl_dg_lowpass_filter_priv.h new file mode 100644 index 0000000000..a2b9052f2a --- /dev/null +++ b/core/zero/gkyl_dg_lowpass_filter_priv.h @@ -0,0 +1,63 @@ +#pragma once + +// Private header for dg_lowpass_filter updater, not for direct use in user code. + +#include +#include +#include +#include + +// Primary struct in this updater. +struct gkyl_dg_lowpass_filter { + int ndim; // Dimensionality of the field. + bool use_gpu; // Whether to use the GPU. + int dir; // Direction along which to filter. + int half_width; // Stencil half-width M (stencil spans 2M+1 cells). + int num_basis; // Number of DG coefficients per cell. + struct gkyl_rect_grid grid; // Grid the field is defined on. + struct gkyl_range range; // Range to filter in. + bool reflect_lo; // Whether to reflect the stencil at the lower edge. + bool reflect_up; // Whether to reflect the stencil at the upper edge. + double *weights; // 2M+1 filter weights, normalized to sum to 1. + double *sign_mirror; // Per-coefficient sign for a mirrored donor. + double *sign_plain; // All ones, for a donor that was not mirrored. + double *weights_cu; // Device copy of weights. + double *sign_mirror_cu; // Device copy of sign_mirror. + double *sign_plain_cu; // Device copy of sign_plain. +}; + +#ifdef GKYL_HAVE_CUDA +// Declaration of cuda device function. +void gkyl_dg_lowpass_filter_advance_cu(gkyl_dg_lowpass_filter *up, + struct gkyl_array *GKYL_RESTRICT fdo, struct gkyl_array *GKYL_RESTRICT ftar); +#endif + +GKYL_CU_DH +static inline int +dg_lpf_mirror_idx(int idx, int lo, int up, bool *mirrored) +{ + // Reflect an out-of-range index back in about the outer cell faces. + int num_reflections = 0; + while (idx < lo || idx > up) { + if (idx < lo) { idx = 2*lo - 1 - idx; num_reflections++; } + if (idx > up) { idx = 2*up + 1 - idx; num_reflections++; } + } + *mirrored = num_reflections % 2 == 1; + return idx; +} + +static void +dg_lpf_calc_weights(int half_width, double fc, double *weights) +{ + // Sinc (cutoff fc in cycles/cell) times a Blackman window, normalized. + int M = half_width; + double wsum = 0.0; + for (int k=-M; k #include #include +#include #include #include @@ -65,6 +66,7 @@ gk_field_fem_projection_par_phi_ts_3x(gkyl_gyrokinetic_app *app, struct gk_field gkyl_array_copy_range_to_range(field->rho_c_global_dg, field->rho_c_global_dg, &app->global_lower_ghost[par_dir], &app->global_upper_skin[par_dir]); gkyl_bc_twistshift_advance(field->bc_ts_lo, field->rho_c_global_dg, field->rho_c_global_dg); + // Fill upper parallel boundary ghost with skin boundary value. gkyl_bc_basic_gyrokinetic_advance(field->gfss_bc_op_core_up, field->bc_buffer, field->rho_c_global_dg); // Smooth the the DG array. @@ -227,7 +229,7 @@ gk_field_2x3x_fill_fem_parproj_bias_lines(struct gkyl_gyrokinetic_app *app, stru } static void -gk_field_2x3x_add_TS_updaters(struct gkyl_gyrokinetic_app *app, struct gk_field *f, struct gkyl_poisson_bc *poisson_bcs) +gk_field_2x3x_add_TS_updaters(struct gkyl_gyrokinetic_app *app, struct gk_field *gkf, struct gkyl_poisson_bc *poisson_bcs) { // Allocation ranges and updaters for TS field solve. @@ -239,19 +241,20 @@ gk_field_2x3x_add_TS_updaters(struct gkyl_gyrokinetic_app *app, struct gk_field fem_parproj_bc_rho_core = GKYL_FEM_PARPROJ_PERIODIC; fem_parproj_bc_phi_core = GKYL_FEM_PARPROJ_PERIODIC; - f->fem_projection_par_rho_func = gk_field_fem_projection_par_rho_ts_2x; - f->fem_projection_par_phi_func = gk_field_fem_projection_par_phi_ts_2x; + gkf->fem_projection_par_rho_func = gk_field_fem_projection_par_rho_ts_2x; + gkf->fem_projection_par_phi_func = gk_field_fem_projection_par_phi_ts_2x; } else if (app->cdim == 3) { - // Here fem_parproj_bc_rho is not actually relevant because we don't use f->fem_parproj_rho. + // Here fem_parproj_bc_rho is not actually relevant because we don't use gkf->fem_parproj_rho. fem_parproj_bc_rho_core = 0; fem_parproj_bc_phi_core = GKYL_FEM_PARPROJ_DIRICHLET_GHOST; - f->fem_projection_par_rho_func = gk_field_fem_projection_par; - f->fem_projection_par_phi_func = gk_field_fem_projection_par_phi_ts_3x; + gkf->fem_projection_par_rho_func = gk_field_fem_projection_par; + gkf->fem_projection_par_phi_func = gk_field_fem_projection_par_phi_ts_3x; + int num_ghost[] = {1, 1, 1}; int par_dir = app->cdim-1; // Parallel direction index. - int ghost[] = {1, 1, 1}; + // TS BC updater for lower edge. struct gkyl_bc_twistshift_inp T_LU_lo = { .bc_dir = par_dir, @@ -260,10 +263,13 @@ gk_field_2x3x_add_TS_updaters(struct gkyl_gyrokinetic_app *app, struct gk_field .edge = GKYL_LOWER_EDGE, .cdim = app->cdim, .bcdir_ext_update_r = &app->global_par_ext, - .num_ghost = ghost, // one ghost per config direction + .num_ghost = num_ghost, // one ghost per config direction .basis = &app->basis, .grid = &app->grid, .use_gpu = app->use_gpu, + .upsample_factor = app->ts_upsample_factor, + .filter_half_width = app->ts_filter_half_width, + .filter_cutoff_wavelength = app->ts_filter_cutoff_wavelength, }; if (app->gk_geom->geometry_id == GKYL_GEOMETRY_TOKAMAK) T_LU_lo.shift_dg = app->delta_ts_x_lo; @@ -271,7 +277,7 @@ gk_field_2x3x_add_TS_updaters(struct gkyl_gyrokinetic_app *app, struct gk_field T_LU_lo.shift_func = app->gk_geom->parallel_lower_bc_shift_func; T_LU_lo.shift_func_ctx = app->gk_geom->parallel_lower_bc_shift_ctx; } - f->bc_ts_lo = gkyl_bc_twistshift_inew(&T_LU_lo); + gkf->bc_ts_lo = gkyl_bc_twistshift_inew(&T_LU_lo); // TS BC updater for upper edge. struct gkyl_bc_twistshift_inp T_UL_up = { @@ -281,10 +287,13 @@ gk_field_2x3x_add_TS_updaters(struct gkyl_gyrokinetic_app *app, struct gk_field .edge = GKYL_UPPER_EDGE, .cdim = app->cdim, .bcdir_ext_update_r = &app->global_par_ext, - .num_ghost = ghost, // one ghost per config direction + .num_ghost = num_ghost, // one ghost per config direction .basis = &app->basis, .grid = &app->grid, .use_gpu = app->use_gpu, + .upsample_factor = app->ts_upsample_factor, + .filter_half_width = app->ts_filter_half_width, + .filter_cutoff_wavelength = app->ts_filter_cutoff_wavelength, }; if (app->gk_geom->geometry_id == GKYL_GEOMETRY_TOKAMAK) T_UL_up.shift_dg = app->delta_ts_x_up; @@ -292,35 +301,35 @@ gk_field_2x3x_add_TS_updaters(struct gkyl_gyrokinetic_app *app, struct gk_field T_UL_up.shift_func = app->gk_geom->parallel_upper_bc_shift_func; T_UL_up.shift_func_ctx = app->gk_geom->parallel_upper_bc_shift_ctx; } - f->bc_ts_up = gkyl_bc_twistshift_inew(&T_UL_up); + gkf->bc_ts_up = gkyl_bc_twistshift_inew(&T_UL_up); long buff_sz = app->global_lower_ghost[par_dir].volume; - f->bc_buffer = mkarr(app->use_gpu, app->basis.num_basis, buff_sz); + gkf->bc_buffer = mkarr(app->use_gpu, app->basis.num_basis, buff_sz); - f->gfss_bc_op_core_up = gkyl_bc_basic_gyrokinetic_new(par_dir, GKYL_UPPER_EDGE, GKYL_BC_GK_FIELD_BOUNDARY_VALUE, + gkf->gfss_bc_op_core_up = gkyl_bc_basic_gyrokinetic_new(par_dir, GKYL_UPPER_EDGE, GKYL_BC_GK_FIELD_BOUNDARY_VALUE, app->basis_on_dev, &app->global_upper_skin[par_dir], &app->global_upper_ghost[par_dir], app->basis.num_basis, app->cdim, app->use_gpu); - f->gfss_bc_op_core_lo = gkyl_bc_basic_gyrokinetic_new(par_dir, GKYL_LOWER_EDGE, GKYL_BC_GK_FIELD_BOUNDARY_VALUE, + gkf->gfss_bc_op_core_lo = gkyl_bc_basic_gyrokinetic_new(par_dir, GKYL_LOWER_EDGE, GKYL_BC_GK_FIELD_BOUNDARY_VALUE, app->basis_on_dev, &app->global_lower_skin[par_dir], &app->global_lower_ghost[par_dir], app->basis.num_basis, app->cdim, app->use_gpu); } // Parallel smoother for the charge density. - f->fem_parproj_rho_core = gkyl_fem_parproj_new(&app->global, &app->grid, &app->basis, + gkf->fem_parproj_rho_core = gkyl_fem_parproj_new(&app->global, &app->grid, &app->basis, fem_parproj_bc_rho_core, 0, 0, 0, app->use_gpu); // Fill bias line list for fem_parproj_phi. - gk_field_2x3x_fill_fem_parproj_bias_lines(app, f, poisson_bcs); + gk_field_2x3x_fill_fem_parproj_bias_lines(app, gkf, poisson_bcs); // Parallel smoother for the potential. - f->fem_parproj_phi_core = gkyl_fem_parproj_new(&app->global, &app->grid, &app->basis, - fem_parproj_bc_phi_core, &f->fem_parproj_bias_line_list, 0, 0, app->use_gpu); + gkf->fem_parproj_phi_core = gkyl_fem_parproj_new(&app->global, &app->grid, &app->basis, + fem_parproj_bc_phi_core, &gkf->fem_parproj_bias_line_list, 0, 0, app->use_gpu); } static void -gk_field_2x3x_add_IWL_updaters(struct gkyl_gyrokinetic_app *app, struct gk_field *f, struct gkyl_poisson_bc *poisson_bcs) +gk_field_2x3x_add_IWL_updaters(struct gkyl_gyrokinetic_app *app, struct gk_field *gkf, struct gkyl_poisson_bc *poisson_bcs) { // Allocation ranges and updaters for IWL field solve. @@ -334,18 +343,18 @@ gk_field_2x3x_add_IWL_updaters(struct gkyl_gyrokinetic_app *app, struct gk_field fem_parproj_bc_phi_core = GKYL_FEM_PARPROJ_PERIODIC; fem_parproj_bc_phi_sol = GKYL_FEM_PARPROJ_NONE; - f->fem_projection_par_rho_func = gk_field_fem_projection_par_rho_iwl_2x; - f->fem_projection_par_phi_func = gk_field_fem_projection_par_phi_iwl_2x; + gkf->fem_projection_par_rho_func = gk_field_fem_projection_par_rho_iwl_2x; + gkf->fem_projection_par_phi_func = gk_field_fem_projection_par_phi_iwl_2x; } else if (app->cdim == 3) { - // Here fem_parproj_bc_rho is not actually relevant because we don't use f->fem_parproj_rho. + // Here fem_parproj_bc_rho is not actually relevant because we don't use gkf->fem_parproj_rho. fem_parproj_bc_rho_core = 0; fem_parproj_bc_rho_sol = 0; fem_parproj_bc_phi_core = GKYL_FEM_PARPROJ_DIRICHLET_GHOST; fem_parproj_bc_phi_sol = GKYL_FEM_PARPROJ_DIRICHLET_SKIN; - f->fem_projection_par_rho_func = gk_field_fem_projection_par; - f->fem_projection_par_phi_func = gk_field_fem_projection_par_phi_iwl_3x; + gkf->fem_projection_par_rho_func = gk_field_fem_projection_par; + gkf->fem_projection_par_phi_func = gk_field_fem_projection_par_phi_iwl_3x; int par_dir = app->cdim-1; // Parallel direction index. // TS BC updater for up to low TS for the lower edge. This sets ghost_L = T_LU(ghost_L). @@ -361,6 +370,9 @@ gk_field_2x3x_add_IWL_updaters(struct gkyl_gyrokinetic_app *app, struct gk_field .basis = &app->basis, .grid = &app->grid, .use_gpu = app->use_gpu, + .upsample_factor = app->ts_upsample_factor, + .filter_half_width = app->ts_filter_half_width, + .filter_cutoff_wavelength = app->ts_filter_cutoff_wavelength, }; if (app->gk_geom->geometry_id == GKYL_GEOMETRY_TOKAMAK) T_LU_lo.shift_dg = app->delta_ts_x_lo; @@ -368,30 +380,30 @@ gk_field_2x3x_add_IWL_updaters(struct gkyl_gyrokinetic_app *app, struct gk_field T_LU_lo.shift_func = app->gk_geom->parallel_lower_bc_shift_func; T_LU_lo.shift_func_ctx = app->gk_geom->parallel_lower_bc_shift_ctx; } - f->bc_ts_lo = gkyl_bc_twistshift_inew(&T_LU_lo); + gkf->bc_ts_lo = gkyl_bc_twistshift_inew(&T_LU_lo); long buff_sz = GKYL_MAX2(app->global_lower_ghost_par_sol.volume, app->global_lower_ghost_par_core.volume); - f->bc_buffer = mkarr(app->use_gpu, app->basis.num_basis, buff_sz); + gkf->bc_buffer = mkarr(app->use_gpu, app->basis.num_basis, buff_sz); - f->gfss_bc_op_core_up = gkyl_bc_basic_gyrokinetic_new(par_dir, GKYL_UPPER_EDGE, GKYL_BC_GK_FIELD_BOUNDARY_VALUE, + gkf->gfss_bc_op_core_up = gkyl_bc_basic_gyrokinetic_new(par_dir, GKYL_UPPER_EDGE, GKYL_BC_GK_FIELD_BOUNDARY_VALUE, app->basis_on_dev, &app->global_upper_skin_par_core, &app->global_upper_ghost_par_core, app->basis.num_basis, app->cdim, app->use_gpu); } // Parallel smoother for the charge density. - f->fem_parproj_rho_core = gkyl_fem_parproj_new(&app->global_core, &app->grid, &app->basis, + gkf->fem_parproj_rho_core = gkyl_fem_parproj_new(&app->global_core, &app->grid, &app->basis, fem_parproj_bc_rho_core, 0, 0, 0, app->use_gpu); - f->fem_parproj_rho_sol = gkyl_fem_parproj_new(&app->global_sol, &app->grid, &app->basis, + gkf->fem_parproj_rho_sol = gkyl_fem_parproj_new(&app->global_sol, &app->grid, &app->basis, fem_parproj_bc_rho_sol, 0, 0, 0, app->use_gpu); // Fill bias line list for fem_parproj_phi. - gk_field_2x3x_fill_fem_parproj_bias_lines(app, f, poisson_bcs); + gk_field_2x3x_fill_fem_parproj_bias_lines(app, gkf, poisson_bcs); // Parallel smoother for the potential. - f->fem_parproj_phi_core = gkyl_fem_parproj_new(&app->global_core, &app->grid, &app->basis, - fem_parproj_bc_phi_core, &f->fem_parproj_bias_line_list, 0, 0, app->use_gpu); - f->fem_parproj_phi_sol = gkyl_fem_parproj_new(&app->global_sol, &app->grid, &app->basis, - fem_parproj_bc_phi_sol, &f->fem_parproj_bias_line_list, 0, 0, app->use_gpu); + gkf->fem_parproj_phi_core = gkyl_fem_parproj_new(&app->global_core, &app->grid, &app->basis, + fem_parproj_bc_phi_core, &gkf->fem_parproj_bias_line_list, 0, 0, app->use_gpu); + gkf->fem_parproj_phi_sol = gkyl_fem_parproj_new(&app->global_sol, &app->grid, &app->basis, + fem_parproj_bc_phi_sol, &gkf->fem_parproj_bias_line_list, 0, 0, app->use_gpu); } diff --git a/gyrokinetic/apps/gk_species.c b/gyrokinetic/apps/gk_species.c index 873236eda2..0f2a090b04 100644 --- a/gyrokinetic/apps/gk_species.c +++ b/gyrokinetic/apps/gk_species.c @@ -660,94 +660,96 @@ gk_species_write_L2norm_static(gkyl_gyrokinetic_app* app, struct gk_species *gks } static void -gk_species_release_dynamic(const gkyl_gyrokinetic_app* app, const struct gk_species *s) +gk_species_release_dynamic(const gkyl_gyrokinetic_app* app, const struct gk_species *gks) { // Release various arrays and objects for a dynamic species. - gkyl_array_release(s->f1); - gkyl_array_release(s->fnew); - gkyl_array_release(s->bc_buffer); - gkyl_array_release(s->bc_buffer_lo_fixed); - gkyl_array_release(s->bc_buffer_up_fixed); + gkyl_array_release(gks->f1); + gkyl_array_release(gks->fnew); + gkyl_array_release(gks->bc_buffer); + gkyl_array_release(gks->bc_buffer_lo_fixed); + gkyl_array_release(gks->bc_buffer_up_fixed); - if (s->info.write_omega_cfl) { - gkyl_array_release(s->cflrate_ho); + if (gks->info.write_omega_cfl) { + gkyl_array_release(gks->cflrate_ho); } + int par_dir = app->cdim-1; // Parallel direction index. + // Copy BCs are allocated by default. Need to free. for (int d=0; dcdim; ++d) { - if (s->lower_bc[d].type == GKYL_BC_GK_SPECIES_SHEATH) { - gkyl_bc_sheath_gyrokinetic_release(s->bc_sheath_lo); + if (gks->lower_bc[d].type == GKYL_BC_GK_SPECIES_SHEATH) { + gkyl_bc_sheath_gyrokinetic_release(gks->bc_sheath_lo); } - else if (s->lower_bc[d].type == GKYL_BC_GK_SPECIES_TWISTSHIFT) { - gkyl_bc_twistshift_release(s->bc_ts_lo); + else if (gks->lower_bc[d].type == GKYL_BC_GK_SPECIES_TWISTSHIFT) { + gkyl_bc_twistshift_release(gks->bc_ts_lo); } - else if ( (s->lower_bc[d].type == GKYL_BC_GK_SPECIES_COPY) || - (s->lower_bc[d].type == GKYL_BC_GK_SPECIES_ABSORB) || - (s->lower_bc[d].type == GKYL_BC_GK_SPECIES_REFLECT) || - (s->lower_bc[d].type == GKYL_BC_GK_SPECIES_FIXED_FUNC) ) { - gkyl_bc_basic_gyrokinetic_release(s->bc_lo[d]); + else if ( (gks->lower_bc[d].type == GKYL_BC_GK_SPECIES_COPY) || + (gks->lower_bc[d].type == GKYL_BC_GK_SPECIES_ABSORB) || + (gks->lower_bc[d].type == GKYL_BC_GK_SPECIES_REFLECT) || + (gks->lower_bc[d].type == GKYL_BC_GK_SPECIES_FIXED_FUNC) ) { + gkyl_bc_basic_gyrokinetic_release(gks->bc_lo[d]); } - if (s->upper_bc[d].type == GKYL_BC_GK_SPECIES_SHEATH) { - gkyl_bc_sheath_gyrokinetic_release(s->bc_sheath_up); + if (gks->upper_bc[d].type == GKYL_BC_GK_SPECIES_SHEATH) { + gkyl_bc_sheath_gyrokinetic_release(gks->bc_sheath_up); } - else if (s->upper_bc[d].type == GKYL_BC_GK_SPECIES_TWISTSHIFT) { - gkyl_bc_twistshift_release(s->bc_ts_up); + else if (gks->upper_bc[d].type == GKYL_BC_GK_SPECIES_TWISTSHIFT) { + gkyl_bc_twistshift_release(gks->bc_ts_up); } - else if ( (s->upper_bc[d].type == GKYL_BC_GK_SPECIES_COPY) || - (s->upper_bc[d].type == GKYL_BC_GK_SPECIES_ABSORB) || - (s->upper_bc[d].type == GKYL_BC_GK_SPECIES_REFLECT) || - (s->upper_bc[d].type == GKYL_BC_GK_SPECIES_FIXED_FUNC) ) { - gkyl_bc_basic_gyrokinetic_release(s->bc_up[d]); + else if ( (gks->upper_bc[d].type == GKYL_BC_GK_SPECIES_COPY) || + (gks->upper_bc[d].type == GKYL_BC_GK_SPECIES_ABSORB) || + (gks->upper_bc[d].type == GKYL_BC_GK_SPECIES_REFLECT) || + (gks->upper_bc[d].type == GKYL_BC_GK_SPECIES_FIXED_FUNC) ) { + gkyl_bc_basic_gyrokinetic_release(gks->bc_up[d]); } } if (app->gk_geom->has_LCFS && app->cdim == 3) { // Free twishift memory. - gkyl_bc_twistshift_release(s->bc_ts_lo); - gkyl_bc_twistshift_release(s->bc_ts_up); + gkyl_bc_twistshift_release(gks->bc_ts_lo); + gkyl_bc_twistshift_release(gks->bc_ts_up); } if (app->use_gpu) { - gkyl_cu_free(s->omega_cfl); - gkyl_cu_free(s->m0_max); + gkyl_cu_free(gks->omega_cfl); + gkyl_cu_free(gks->m0_max); } else { - gkyl_free(s->omega_cfl); - gkyl_free(s->m0_max); + gkyl_free(gks->omega_cfl); + gkyl_free(gks->m0_max); } // Release integrated moment memory. - gk_species_moment_release(app, &s->integ_moms); + gk_species_moment_release(app, &gks->integ_moms); // Release integrated diag memory. - gkyl_dynvec_release(s->integ_diag); + gkyl_dynvec_release(gks->integ_diag); if (app->use_gpu) { - gkyl_cu_free(s->red_integ_diag); - gkyl_cu_free(s->red_integ_diag_global); + gkyl_cu_free(gks->red_integ_diag); + gkyl_cu_free(gks->red_integ_diag_global); } else { - gkyl_free(s->red_integ_diag); - gkyl_free(s->red_integ_diag_global); + gkyl_free(gks->red_integ_diag); + gkyl_free(gks->red_integ_diag_global); } // Release L2 norm memory. - gkyl_array_integrate_release(s->integ_wfsq_op); - gkyl_dynvec_release(s->L2norm); + gkyl_array_integrate_release(gks->integ_wfsq_op); + gkyl_dynvec_release(gks->L2norm); if (app->use_gpu) { - gkyl_cu_free(s->L2norm_local); - gkyl_cu_free(s->L2norm_global); + gkyl_cu_free(gks->L2norm_local); + gkyl_cu_free(gks->L2norm_global); } else { - gkyl_free(s->L2norm_local); - gkyl_free(s->L2norm_global); + gkyl_free(gks->L2norm_local); + gkyl_free(gks->L2norm_global); } - if (s->info.time_rate_diagnostics) { + if (gks->info.time_rate_diagnostics) { // Free df/dt diagnostics memory. - gkyl_array_release(s->fdot_mom_old); - gkyl_array_release(s->fdot_mom_new); - gkyl_dynvec_release(s->fdot_integ_diag); + gkyl_array_release(gks->fdot_mom_old); + gkyl_array_release(gks->fdot_mom_new); + gkyl_dynvec_release(gks->fdot_integ_diag); } } @@ -763,14 +765,12 @@ gk_species_init_dynamic(struct gkyl_gk *gk_app_inp, struct gkyl_gyrokinetic_app int cdim = app->cdim, vdim = gks->info.vdim; int pdim = cdim+vdim; - int ghost[GKYL_MAX_DIM]; - + int num_ghost[GKYL_MAX_DIM]; for (int d=0; ddt_omegaH = DBL_MIN; @@ -826,7 +826,7 @@ gk_species_init_dynamic(struct gkyl_gk *gk_app_inp, struct gkyl_gyrokinetic_app gks->L2norm = gkyl_dynvec_new(GKYL_DOUBLE, 1); // L2 norm. gks->is_first_L2norm_write_call = true; - int par_dir = app->cdim-1; // The last direction is the parallel one. + int par_dir = app->cdim-1; // Parallel direction index. if (gk_app_inp->geometry.has_LCFS || (gks->lower_bc[par_dir].type == GKYL_BC_GK_SPECIES_TWISTSHIFT || gks->upper_bc[par_dir].type == GKYL_BC_GK_SPECIES_TWISTSHIFT)) { // Make the parallel direction periodic so that we sync before applying TS BC. @@ -870,6 +870,19 @@ gk_species_init_dynamic(struct gkyl_gk *gk_app_inp, struct gkyl_gyrokinetic_app gks->bc_buffer_up_fixed = need_bc_buffer_up_fixed? mkarr(app->use_gpu, gks->basis.num_basis, buff_sz) : mkarr(app->use_gpu, 1, 1); + if (gks->lower_bc[par_dir].type == GKYL_BC_GK_SPECIES_TWISTSHIFT && + gks->upper_bc[par_dir].type == GKYL_BC_GK_SPECIES_TWISTSHIFT) { + // Local range extended in the BC dir, on the coarse grid. + int lower_bcdir_ext[pdim], upper_bcdir_ext[pdim]; + for (int i=0; ilocal.lower[i]; + upper_bcdir_ext[i] = gks->local.upper[i]; + } + lower_bcdir_ext[par_dir] = gks->local_ext.lower[par_dir]; + upper_bcdir_ext[par_dir] = gks->local_ext.upper[par_dir]; + gkyl_sub_range_init(&gks->bc_ts_local_par_ext, &gks->local_ext, lower_bcdir_ext, upper_bcdir_ext); + } + for (int d=0; dlower_bc[d].type == GKYL_BC_GK_SPECIES_SHEATH) { @@ -886,11 +899,14 @@ gk_species_init_dynamic(struct gkyl_gk *gk_app_inp, struct gkyl_gyrokinetic_app .shear_dir = 0, // shift varies with x. .edge = GKYL_LOWER_EDGE, .cdim = cdim, - .bcdir_ext_update_r = &gks->local_par_ext, - .num_ghost = ghost, + .bcdir_ext_update_r = &gks->bc_ts_local_par_ext, + .num_ghost = num_ghost, .basis = &gks->basis, .grid = &gks->grid, .use_gpu = app->use_gpu, + .upsample_factor = app->ts_upsample_factor, + .filter_half_width = app->ts_filter_half_width, + .filter_cutoff_wavelength = app->ts_filter_cutoff_wavelength, }; if (app->gk_geom->geometry_id == GKYL_GEOMETRY_TOKAMAK) tsinp.shift_dg = app->delta_ts_x_lo; @@ -946,11 +962,14 @@ gk_species_init_dynamic(struct gkyl_gk *gk_app_inp, struct gkyl_gyrokinetic_app .shear_dir = 0, // shift varies with x. .edge = GKYL_UPPER_EDGE, .cdim = cdim, - .bcdir_ext_update_r = &gks->local_par_ext, - .num_ghost = ghost, + .bcdir_ext_update_r = &gks->bc_ts_local_par_ext, + .num_ghost = num_ghost, .basis = &gks->basis, .grid = &gks->grid, .use_gpu = app->use_gpu, + .upsample_factor = app->ts_upsample_factor, + .filter_half_width = app->ts_filter_half_width, + .filter_cutoff_wavelength = app->ts_filter_cutoff_wavelength, }; if (app->gk_geom->geometry_id == GKYL_GEOMETRY_TOKAMAK) tsinp.shift_dg = app->delta_ts_x_up; @@ -1000,10 +1019,13 @@ gk_species_init_dynamic(struct gkyl_gk *gk_app_inp, struct gkyl_gyrokinetic_app .edge = GKYL_LOWER_EDGE, .cdim = cdim, .bcdir_ext_update_r = &gks->local_par_ext_core, - .num_ghost = ghost, + .num_ghost = num_ghost, .basis = &gks->basis, .grid = &gks->grid, .use_gpu = app->use_gpu, + .upsample_factor = app->ts_upsample_factor, + .filter_half_width = app->ts_filter_half_width, + .filter_cutoff_wavelength = app->ts_filter_cutoff_wavelength, }; if (app->gk_geom->geometry_id == GKYL_GEOMETRY_TOKAMAK) tsinp_lo.shift_dg = app->delta_ts_x_lo; @@ -1020,10 +1042,13 @@ gk_species_init_dynamic(struct gkyl_gk *gk_app_inp, struct gkyl_gyrokinetic_app .edge = GKYL_UPPER_EDGE, .cdim = cdim, .bcdir_ext_update_r = &gks->local_par_ext_core, - .num_ghost = ghost, + .num_ghost = num_ghost, .basis = &gks->basis, .grid = &gks->grid, .use_gpu = app->use_gpu, + .upsample_factor = app->ts_upsample_factor, + .filter_half_width = app->ts_filter_half_width, + .filter_cutoff_wavelength = app->ts_filter_cutoff_wavelength, }; if (app->gk_geom->geometry_id == GKYL_GEOMETRY_TOKAMAK) tsinp_up.shift_dg = app->delta_ts_x_up; diff --git a/gyrokinetic/apps/gkyl_gyrokinetic.h b/gyrokinetic/apps/gkyl_gyrokinetic.h index 1d137b0963..780cba59c4 100644 --- a/gyrokinetic/apps/gkyl_gyrokinetic.h +++ b/gyrokinetic/apps/gkyl_gyrokinetic.h @@ -250,6 +250,11 @@ struct gkyl_gyrokinetic_geometry { void *parallel_lower_bc_shift_ctx; // Context for parallel_lower_bc_shift_func. void *parallel_upper_bc_shift_ctx; // Context for parallel_upper_bc_shift_func. + // Twist-shift anti-aliasing filter. + int ts_upsample_factor; // Supersampling factor along the shear direction. + int ts_filter_half_width; // Filter stencil half-width in cells of the simulation grid (0 = off). + double ts_filter_cutoff_wavelength; // Filter cutoff wavelength (use 2*dx to cut at the coarse Nyquist). + struct gkyl_efit_inp efit_info; // Context with RZ data such as efit file for a tokamak or mirror. struct gkyl_tok_geo_grid_inp tok_grid_info; // Context for tokamak geometry with computational domain info. struct gkyl_mirror_geo_grid_inp mirror_grid_info; // Context for mirror geometry with computational domain info. diff --git a/gyrokinetic/apps/gkyl_gyrokinetic_priv.h b/gyrokinetic/apps/gkyl_gyrokinetic_priv.h index e8d0955901..59ae31615c 100644 --- a/gyrokinetic/apps/gkyl_gyrokinetic_priv.h +++ b/gyrokinetic/apps/gkyl_gyrokinetic_priv.h @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -1097,6 +1098,14 @@ struct gk_species { struct gkyl_range local_upper_skin_par_sol , local_upper_ghost_par_sol; // GK IWL sims need a core range extended in z, and a TS BC updater. struct gkyl_range local_par_ext_core; // Core range extended in parallel direction. + // Objects used in IWL simulations and TS BCs. + struct gkyl_rect_grid bc_ts_grid; // Higher resolution grid for TS BC. + struct gkyl_range bc_ts_local_ext, bc_ts_local; // Higher resolution ranges for TS BC. + struct gkyl_range bc_ts_local_lower_skin_par, bc_ts_local_upper_skin_par; // Parallel skin for TS BC. + struct gkyl_range bc_ts_local_lower_ghost_par, bc_ts_local_upper_ghost_par; // Parallel ghost for TS BC. + struct gkyl_dg_interpolate *bc_ts_prolong, *bc_ts_coarsen; // Interpolation operators for TS BC. + struct gkyl_array *bc_ts_buffer_fine, *bc_ts_buffer_coar; // Buffer for TS BCs. + struct gkyl_range bc_ts_local_par_ext; // Range extended in parallel direction for TS BC. struct gkyl_bc_twistshift *bc_ts_lo, *bc_ts_up; struct gk_proj proj_init; // Projector for initial conditions. @@ -1393,7 +1402,14 @@ struct gk_field { void (*calc_energy_dt_func)(gkyl_gyrokinetic_app *app, const struct gk_field *field, double dt, double *energy_reduced); // Objects used in IWL simulations and TS BCs. - struct gkyl_bc_twistshift *bc_ts_lo, *bc_ts_up; + struct gkyl_rect_grid bc_ts_grid; // Higher resolution grid for TS BC. + struct gkyl_range bc_ts_global_ext, bc_ts_global; // Higher resolution ranges for TS BC. + struct gkyl_range bc_ts_global_lower_skin_par, bc_ts_global_upper_skin_par; // Parallel skin for TS BC. + struct gkyl_range bc_ts_global_lower_ghost_par, bc_ts_global_upper_ghost_par; // Parallel ghost for TS BC. + struct gkyl_dg_interpolate *bc_ts_prolong, *bc_ts_coarsen; // Interpolation operators for TS BC. + struct gkyl_array *bc_ts_buffer_fine, *bc_ts_buffer_coar; // Buffer for TS BCs. + struct gkyl_range bc_ts_global_par_ext; // Range extended in parallel direction for TS BC. + struct gkyl_bc_twistshift *bc_ts_lo, *bc_ts_up; // Fills z-ghosts with TS BC. struct gkyl_bc_basic_gyrokinetic *gfss_bc_op_core_up; // Fills upper core z-ghost with skin boundary value. struct gkyl_bc_basic_gyrokinetic *gfss_bc_op_core_lo; // Fills lower core z-ghost with skin boundary value. struct gkyl_array *bc_buffer; // Buffer for bc_basic. @@ -1427,6 +1443,9 @@ struct gkyl_gyrokinetic_app { int cdim; // Configuration space dimensions. int poly_order; // Polynomial order. + int ts_upsample_factor; // Twist-shift supersampling factor (0/1 = none). + int ts_filter_half_width; // Twist-shift filter half-width (0 = off). + double ts_filter_cutoff_wavelength; // Twist-shift filter cutoff wavelength. double tcurr; // Current time. double cfl; // CFL number. double cfl_omegaH; // CFL number used for omega_H. diff --git a/gyrokinetic/apps/gyrokinetic.c b/gyrokinetic/apps/gyrokinetic.c index 5cf1ff2c99..9799c0ab21 100644 --- a/gyrokinetic/apps/gyrokinetic.c +++ b/gyrokinetic/apps/gyrokinetic.c @@ -172,6 +172,9 @@ gkyl_gyrokinetic_app_new_geom(struct gkyl_gk *gk) int cdim = app->cdim = gk->cdim; int poly_order = app->poly_order = gk->poly_order; + app->ts_upsample_factor = gk->geometry.ts_upsample_factor; + app->ts_filter_half_width = gk->geometry.ts_filter_half_width; + app->ts_filter_cutoff_wavelength = gk->geometry.ts_filter_cutoff_wavelength; int ns = app->num_species = gk->num_species; int neuts = app->num_neut_species = gk->num_neut_species; @@ -1245,7 +1248,7 @@ gyrokinetic_app_write_ts_shift_mapc2p(struct gkyl_gyrokinetic_app *app) for (int eI = 0; eI < 2; eI++) { int ghost[] = {1, 1, 1}; // TS BC updater. - struct gkyl_bc_twistshift_inp ts_inp = { + struct gkyl_twistshift_dg_inp ts_inp = { .bc_dir = par_dir, .shift_dir = 1, // y shift. .shear_dir = 0, // shift varies with x. @@ -1259,10 +1262,10 @@ gyrokinetic_app_write_ts_shift_mapc2p(struct gkyl_gyrokinetic_app *app) .shift_func_ctx = eI == 0? app->gk_geom->parallel_lower_bc_shift_ctx : app->gk_geom->parallel_upper_bc_shift_ctx, .use_gpu = app->use_gpu, }; - struct gkyl_bc_twistshift *bc_ts_op = gkyl_bc_twistshift_inew(&ts_inp); + struct gkyl_twistshift_dg *bc_ts_op = gkyl_twistshift_dg_inew(&ts_inp); struct gkyl_array *delta_ts_x = eI == 0? app->delta_ts_x_lo : app->delta_ts_x_up; - delta_ts_x = gkyl_bc_twistshift_get_shift_objects(bc_ts_op, + delta_ts_x = gkyl_twistshift_dg_get_shift_objects(bc_ts_op, &app->delta_ts_x_grid, &app->delta_ts_x_rng, &app->delta_ts_x_basis); bool has_LCFS = app->gk_geom->has_LCFS; @@ -1308,7 +1311,7 @@ gyrokinetic_app_write_ts_shift_mapc2p(struct gkyl_gyrokinetic_app *app) gkyl_array_release(delta_ts_x); gkyl_msgpack_data_release(mt_shift); - gkyl_bc_twistshift_release(bc_ts_op); + gkyl_twistshift_dg_release(bc_ts_op); } } diff --git a/gyrokinetic/creg/rt_gk_tcv_core_3x2v_p1.c b/gyrokinetic/creg/rt_gk_tcv_core_3x2v_p1.c index f9ca5a4bb4..1f89d7bed3 100644 --- a/gyrokinetic/creg/rt_gk_tcv_core_3x2v_p1.c +++ b/gyrokinetic/creg/rt_gk_tcv_core_3x2v_p1.c @@ -741,8 +741,6 @@ main(int argc, char **argv) // GK app. struct gkyl_gk app_inp = { - .name = "rt_gk_tcv_core_3x2v_p1", - .cfl_frac = 1.0, .cdim = ctx.cdim, @@ -779,6 +777,9 @@ main(int argc, char **argv) }, }; + // Set app output name from the executable name (argv[0]). + snprintf(app_inp.name, sizeof(app_inp.name), "%s", app_args.app_name); + struct gkyl_gyrokinetic_run_inp run_inp = { .app_inp = app_inp, .time_stepping = { diff --git a/gyrokinetic/creg/rt_gk_tcv_iwl_adapt_source_3x2v_p1.c b/gyrokinetic/creg/rt_gk_tcv_iwl_adapt_source_3x2v_p1.c index 9798bee053..1a9394d9b3 100644 --- a/gyrokinetic/creg/rt_gk_tcv_iwl_adapt_source_3x2v_p1.c +++ b/gyrokinetic/creg/rt_gk_tcv_iwl_adapt_source_3x2v_p1.c @@ -439,10 +439,10 @@ struct gk_app_ctx create_ctx(void) double floor_srcRECY = 1e-10; // Grid parameters (reduced resolution for the regression test, minimal recommended values in comments) - int Nx = 24; // (24) The LCFS is positionned at 1/3 of the domain -> the resolution must be divisible by 3. - int Ny = 4; // (16) + int Nx = 15; // (24) The LCFS is positionned at 1/3 of the domain -> the resolution must be divisible by 3. + int Ny = 8; // (16) int Nz = 8; // (12) - int Nvpar = 8; // (12) + int Nvpar = 4; // (12) int Nmu = 4; // (8) int poly_order = 1; // Velocity box dimensions @@ -822,7 +822,7 @@ main(int argc, char **argv) struct gkyl_gyrokinetic_geometry geometry = { .geometry_id = GKYL_GEOMETRY_MAPC2P, .world = {0.}, - .mapc2p = mapc2p, // mapping of cCOREutational to physical space + .mapc2p = mapc2p, // mapping of computational to physical space .c2p_ctx = &ctx, .bfield_func = bfield_func, // magnetic field .bfield_ctx = &ctx, @@ -832,6 +832,9 @@ main(int argc, char **argv) .parallel_upper_bc_shift_func = bc_shift_func_up, .parallel_lower_bc_shift_ctx = &ctx, .parallel_upper_bc_shift_ctx = &ctx, + .ts_filter_cutoff_wavelength = 2.0*ctx.Lx/ctx.Nx, + .ts_filter_half_width = 1, + .ts_upsample_factor = 2, }; // Parallelism @@ -869,6 +872,7 @@ main(int argc, char **argv) // Set app output name from the executable name (argv[0]). snprintf(app_inp.name, sizeof(app_inp.name), "%s", app_args.app_name); + struct gkyl_gyrokinetic_run_inp run_inp = { .app_inp = app_inp, .time_stepping = { diff --git a/gyrokinetic/gyrokineticlinkobjs.mak b/gyrokinetic/gyrokineticlinkobjs.mak index f5a1de13e5..01af73ed6c 100644 --- a/gyrokinetic/gyrokineticlinkobjs.mak +++ b/gyrokinetic/gyrokineticlinkobjs.mak @@ -1,7 +1,7 @@ # -*- makefile-gmake -*- # Gyrokinetic include objects -GYROKINETIC_INCS = -I../gyrokinetic/$(KERNELS_DIR)/ambi_bolt_potential -I../gyrokinetic/$(KERNELS_DIR)/bgk_gyrokinetic -I../gyrokinetic/$(KERNELS_DIR)/deflate_geo -I../gyrokinetic/$(KERNELS_DIR)/deflate_surf -I../gyrokinetic/$(KERNELS_DIR)/derived_geo -I../gyrokinetic/$(KERNELS_DIR)/dg_diffusion_gyrokinetic -I../gyrokinetic/$(KERNELS_DIR)/fem_parproj -I../gyrokinetic/$(KERNELS_DIR)/fem_poisson_perp -I../gyrokinetic/$(KERNELS_DIR)/gyrokinetic -I../gyrokinetic/$(KERNELS_DIR)/gyrokinetic_pol_density -I../gyrokinetic/$(KERNELS_DIR)/inflate_surf -I../gyrokinetic/$(KERNELS_DIR)/lbo_gyrokinetic -I../gyrokinetic/$(KERNELS_DIR)/neutral -I../gyrokinetic/$(KERNELS_DIR)/positivity_shift_gyrokinetic -I../gyrokinetic/$(KERNELS_DIR)/rad -I../gyrokinetic/$(KERNELS_DIR)/translate_dim -I../gyrokinetic/$(KERNELS_DIR)/twistshift -I../gyrokinetic/apps -I../gyrokinetic/zero +GYROKINETIC_INCS = -I../gyrokinetic/$(KERNELS_DIR)/ambi_bolt_potential -I../gyrokinetic/$(KERNELS_DIR)/bgk_gyrokinetic -I../gyrokinetic/$(KERNELS_DIR)/deflate_geo -I../gyrokinetic/$(KERNELS_DIR)/deflate_surf -I../gyrokinetic/$(KERNELS_DIR)/derived_geo -I../gyrokinetic/$(KERNELS_DIR)/dg_diffusion_gyrokinetic -I../gyrokinetic/$(KERNELS_DIR)/fem_parproj -I../gyrokinetic/$(KERNELS_DIR)/fem_poisson_perp -I../gyrokinetic/$(KERNELS_DIR)/gyrokinetic -I../gyrokinetic/$(KERNELS_DIR)/gyrokinetic_pol_density -I../gyrokinetic/$(KERNELS_DIR)/inflate_surf -I../gyrokinetic/$(KERNELS_DIR)/lbo_gyrokinetic -I../gyrokinetic/$(KERNELS_DIR)/neutral -I../gyrokinetic/$(KERNELS_DIR)/positivity_shift_gyrokinetic -I../gyrokinetic/$(KERNELS_DIR)/rad -I../gyrokinetic/$(KERNELS_DIR)/translate_dim -I../gyrokinetic/$(KERNELS_DIR)/twistshift_dg -I../gyrokinetic/apps -I../gyrokinetic/zero # Gyrokinetic link objects diff --git a/gyrokinetic/ker/twistshift/gkyl_bc_twistshift_gyrokinetic_kernels.h b/gyrokinetic/ker/twistshift_dg/gkyl_twistshift_dg_gyrokinetic_kernels.h similarity index 100% rename from gyrokinetic/ker/twistshift/gkyl_bc_twistshift_gyrokinetic_kernels.h rename to gyrokinetic/ker/twistshift_dg/gkyl_twistshift_dg_gyrokinetic_kernels.h diff --git a/gyrokinetic/ker/twistshift/bc_twistshift_gyrokinetic_ser_2x_p1.c b/gyrokinetic/ker/twistshift_dg/twistshift_dg_gyrokinetic_ser_2x_p1.c similarity index 99% rename from gyrokinetic/ker/twistshift/bc_twistshift_gyrokinetic_ser_2x_p1.c rename to gyrokinetic/ker/twistshift_dg/twistshift_dg_gyrokinetic_ser_2x_p1.c index 40b5c81b90..e0b0111610 100644 --- a/gyrokinetic/ker/twistshift/bc_twistshift_gyrokinetic_ser_2x_p1.c +++ b/gyrokinetic/ker/twistshift_dg/twistshift_dg_gyrokinetic_ser_2x_p1.c @@ -1,4 +1,4 @@ -#include +#include GKYL_CU_DH void twistshift_xlimdg_2x_ser_p1_yshift_p1(double sFac, const double *xLimLo, const double *xLimUp, double yLimLo, double yLimUp, double dyDo, double yOff, const double *ySh, struct gkyl_mat *tsmat) { diff --git a/gyrokinetic/ker/twistshift/bc_twistshift_gyrokinetic_ser_3x2v_p1.c b/gyrokinetic/ker/twistshift_dg/twistshift_dg_gyrokinetic_ser_3x2v_p1.c similarity index 99% rename from gyrokinetic/ker/twistshift/bc_twistshift_gyrokinetic_ser_3x2v_p1.c rename to gyrokinetic/ker/twistshift_dg/twistshift_dg_gyrokinetic_ser_3x2v_p1.c index 623502ca03..8e191a45dc 100644 --- a/gyrokinetic/ker/twistshift/bc_twistshift_gyrokinetic_ser_3x2v_p1.c +++ b/gyrokinetic/ker/twistshift_dg/twistshift_dg_gyrokinetic_ser_3x2v_p1.c @@ -1,4 +1,4 @@ -#include +#include GKYL_CU_DH void twistshift_xlimdg_3x2v_ser_p1_yshift_p1(double sFac, const double *xLimLo, const double *xLimUp, double yLimLo, double yLimUp, double dyDo, double yOff, const double *ySh, struct gkyl_mat *tsmat) { diff --git a/gyrokinetic/ker/twistshift/bc_twistshift_gyrokinetic_ser_3x_p1.c b/gyrokinetic/ker/twistshift_dg/twistshift_dg_gyrokinetic_ser_3x_p1.c similarity index 99% rename from gyrokinetic/ker/twistshift/bc_twistshift_gyrokinetic_ser_3x_p1.c rename to gyrokinetic/ker/twistshift_dg/twistshift_dg_gyrokinetic_ser_3x_p1.c index 3c370244ae..ba3345a92f 100644 --- a/gyrokinetic/ker/twistshift/bc_twistshift_gyrokinetic_ser_3x_p1.c +++ b/gyrokinetic/ker/twistshift_dg/twistshift_dg_gyrokinetic_ser_3x_p1.c @@ -1,4 +1,4 @@ -#include +#include GKYL_CU_DH void twistshift_xlimdg_3x_ser_p1_yshift_p1(double sFac, const double *xLimLo, const double *xLimUp, double yLimLo, double yLimUp, double dyDo, double yOff, const double *ySh, struct gkyl_mat *tsmat) { diff --git a/gyrokinetic/unit/ctest_bc_twistshift.c b/gyrokinetic/unit/ctest_bc_twistshift.c index 707d48c149..05a9536d3a 100644 --- a/gyrokinetic/unit/ctest_bc_twistshift.c +++ b/gyrokinetic/unit/ctest_bc_twistshift.c @@ -1,3829 +1,496 @@ -// Test creation and deallocation of updater that applies the -// twist shift BCs. -// +// Test the bc_twistshift orchestrator: the twist-shift BC on its own, and the +// supersample -> twist-shift -> low-pass filter -> restrict pipeline used to +// de-alias the shift along the shear direction. The sub-cell integrals the +// shift is built from are tested separately, in ctest_twistshift_dg. #include +#include #include -#include -#include -#include -#include #include -#include -#include -#include #include -#include -#include -#include -#include -#include -#include -#include - -// Meta-data for IO -struct test_bc_twistshift_output_meta { - int poly_order; // polynomial order - const char *basis_type; // name of basis functions -}; - -// returned gkyl_array_meta must be freed using gyrokinetic_array_meta_release -static struct gkyl_msgpack_data* -test_bc_twistshift_array_meta_new(struct test_bc_twistshift_output_meta meta) -{ - struct gkyl_msgpack_data *mt = gkyl_malloc(sizeof(*mt)); - - mt->meta_sz = 0; - mpack_writer_t writer; - mpack_writer_init_growable(&writer, &mt->meta, &mt->meta_sz); - - // add some data to mpack - mpack_build_map(&writer); - - mpack_write_cstr(&writer, "polyOrder"); - mpack_write_i64(&writer, meta.poly_order); - - mpack_write_cstr(&writer, "basisType"); - mpack_write_cstr(&writer, meta.basis_type); +#include +#include +#include +#include +#include +#include - mpack_complete_map(&writer); +#include - int status = mpack_writer_destroy(&writer); +static const double ts_lower[] = {-2.0, -1.5, -3.0}; +static const double ts_upper[] = { 2.0, 1.5, 3.0}; +static const int ts_cells[] = {8, 8, 4}; +static const int ts_ndim = 3, ts_cdim = 3, ts_bc_dir = 2; - if (status != mpack_ok) { - free(mt->meta); // we need to use free here as mpack does its own malloc - gkyl_free(mt); - mt = 0; - } +struct ts_ctx { + double offset; // Shift at x=0. + double shear; // Rate at which the shift varies with x (0 = no shear). +}; - return mt; +static void +shift_func(double t, const double *xn, double *fout, void *ctx) +{ + struct ts_ctx *tctx = ctx; + fout[0] = tctx->offset + tctx->shear*xn[0]; } static void -test_bc_twistshift_array_meta_release(struct gkyl_msgpack_data *mt) +init_donor(double t, const double *xn, double *fout, void *ctx) { - if (!mt) return; - MPACK_FREE(mt->meta); - gkyl_free(mt); + double x = xn[0], y = xn[1]; + fout[0] = exp(-y*y/(2.0*0.3*0.3))*(1.0 + 0.5*cos(M_PI*x)); } -struct skin_ghost_ranges { - struct gkyl_range lower_skin[GKYL_MAX_DIM]; - struct gkyl_range lower_ghost[GKYL_MAX_DIM]; - - struct gkyl_range upper_skin[GKYL_MAX_DIM]; - struct gkyl_range upper_ghost[GKYL_MAX_DIM]; +// Everything needed to build and apply the BC on the test grid. +struct ts_setup { + struct gkyl_rect_grid grid; + struct gkyl_basis basis; + struct gkyl_range local, local_ext; + struct gkyl_range update_r; // Local range, extended in bc_dir. + struct gkyl_range ghost_r; // Plane the BC writes. + struct gkyl_range skin_r; // Plane periodicity copies from. + int ghost[GKYL_MAX_DIM]; }; -// Create ghost and skin sub-ranges given a parent range static void -skin_ghost_ranges_init(struct skin_ghost_ranges *sgr, - const struct gkyl_range *parent, const int *ghost) +ts_setup_init(struct ts_setup *s, enum gkyl_edge_loc edge) { - int ndim = parent->ndim; + gkyl_rect_grid_init(&s->grid, ts_ndim, ts_lower, ts_upper, ts_cells); + gkyl_cart_modal_serendip(&s->basis, ts_ndim, 1); + + for (int d=0; dghost[d] = 1; + gkyl_create_grid_ranges(&s->grid, s->ghost, &s->local_ext, &s->local); - for (int d=0; dlower_skin[d], &sgr->lower_ghost[d], - d, GKYL_LOWER_EDGE, parent, ghost); - gkyl_skin_ghost_ranges(&sgr->upper_skin[d], &sgr->upper_ghost[d], - d, GKYL_UPPER_EDGE, parent, ghost); + int lo[GKYL_MAX_DIM], up[GKYL_MAX_DIM]; + for (int d=0; dlocal.lower[d]; + up[d] = s->local.upper[d]; } -} -// Apply periodic BCs along parallel direction -void -apply_periodic_bc(struct gkyl_array *buff, struct gkyl_array *fld, const int dir, const struct skin_ghost_ranges sgr) -{ - gkyl_array_copy_to_buffer(buff->data, fld, &(sgr.lower_skin[dir])); - gkyl_array_copy_from_buffer(fld, buff->data, &(sgr.upper_ghost[dir])); + lo[ts_bc_dir] = s->local_ext.lower[ts_bc_dir]; + up[ts_bc_dir] = s->local_ext.upper[ts_bc_dir]; + gkyl_sub_range_init(&s->update_r, &s->local_ext, lo, up); - gkyl_array_copy_to_buffer(buff->data, fld, &(sgr.upper_skin[dir])); - gkyl_array_copy_from_buffer(fld, buff->data, &(sgr.lower_ghost[dir])); + // The BC reads and writes the same ghost plane; periodicity fills it from + // the skin cell at the other end. + int slo[GKYL_MAX_DIM], sup[GKYL_MAX_DIM]; + for (int d=0; dlocal.lower[d]; + sup[d] = s->local.upper[d]; + } + if (edge == GKYL_LOWER_EDGE) { + gkyl_range_shorten_from_above(&s->ghost_r, &s->update_r, ts_bc_dir, s->ghost[ts_bc_dir]); + slo[ts_bc_dir] = sup[ts_bc_dir] = s->local.upper[ts_bc_dir]; + } + else { + gkyl_range_shorten_from_below(&s->ghost_r, &s->update_r, ts_bc_dir, s->ghost[ts_bc_dir]); + slo[ts_bc_dir] = sup[ts_bc_dir] = s->local.lower[ts_bc_dir]; + } + gkyl_sub_range_init(&s->skin_r, &s->local_ext, slo, sup); } +// Project the donor and apply periodicity along bc_dir. static struct gkyl_array* -mkarr(bool on_gpu, long nc, long size) -{ - struct gkyl_array* a; - if (on_gpu) - a = gkyl_array_cu_dev_new(GKYL_DOUBLE, nc, size); - else - a = gkyl_array_new(GKYL_DOUBLE, nc, size); - return a; -} - -struct test_bc_twistshift_ctx { - double lower[GKYL_MAX_DIM], upper[GKYL_MAX_DIM]; - int cells[GKYL_MAX_DIM]; - double B0; - double vt; - double mass; - enum gkyl_edge_loc edge; -}; - -void -mapc2p(double t, const double *xc, double* GKYL_RESTRICT xp, void *ctx) +ts_donor_new(const struct ts_setup *s) { - xp[0] = xc[0]; xp[1] = xc[1]; xp[2] = xc[2]; -} - -void eval_bfield_3x(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) -{ - double x = xn[0], y = xn[1], z = xn[2]; - - struct test_bc_twistshift_ctx *pars = ctx; - double B0 = pars->B0; - - fout[0] = 0.0; - fout[1] = 0.0; - fout[2] = B0; -} + struct gkyl_array *f = gkyl_array_new(GKYL_DOUBLE, s->basis.num_basis, s->local_ext.volume); + gkyl_array_clear(f, 0.0); -void -shift1_fig6(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) -{ - struct test_bc_twistshift_ctx *pars = ctx; - double Lx[2] = {pars->upper[0]-pars->lower[0], pars->upper[1]-pars->lower[1]}; - double dx[2] = {Lx[0]/pars->cells[0], Lx[1]/pars->cells[1]}; + gkyl_proj_on_basis *proj = gkyl_proj_on_basis_inew(&(struct gkyl_proj_on_basis_inp) { + .grid = &s->grid, .basis = &s->basis, .num_ret_vals = 1, .eval = init_donor, .ctx = NULL }); + gkyl_proj_on_basis_advance(proj, 0.0, &s->local, f); + gkyl_proj_on_basis_release(proj); - fout[0] = 4.0*dx[1]; + gkyl_array_copy_range_to_range(f, f, &s->ghost_r, &s->skin_r); + return f; } -void -shift1m_fig6(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) +// The 1D DG shift on the shear cells of the update range, as the app builds it. +static struct gkyl_array* +ts_shift_dg_new(const struct ts_setup *s, struct ts_ctx *tctx) { - shift1_fig6(t, xn, fout, ctx); - fout[0] *= -1.0; -} + struct gkyl_rect_grid xgrid; + gkyl_rect_grid_init(&xgrid, 1, &ts_lower[0], &ts_upper[0], &ts_cells[0]); + struct gkyl_basis xbasis; + gkyl_cart_modal_serendip(&xbasis, 1, s->basis.poly_order); -void -shift2_fig6(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) -{ - fout[0] = 1.1; -} + struct gkyl_range xrng; + gkyl_range_init(&xrng, 1, (int[]) {s->update_r.lower[0]}, (int[]) {s->update_r.upper[0]}); -void -shift2m_fig6(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) -{ - shift2_fig6(t, xn, fout, ctx); - fout[0] *= -1.0; + struct gkyl_array *shift_dg = gkyl_array_new(GKYL_DOUBLE, xbasis.num_basis, xrng.volume); + gkyl_eval_on_nodes *ev = gkyl_eval_on_nodes_new(&xgrid, &xbasis, 1, shift_func, tctx); + gkyl_eval_on_nodes_advance(ev, 0.0, &xrng, shift_dg); + gkyl_eval_on_nodes_release(ev); + return shift_dg; } -void -init_donor_fig6(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) +// Apply the BC to a fresh donor field and return it. Pass shift_dg to exercise +// the discretized-shift input instead of the shift function. +static struct gkyl_array* +ts_run(const struct ts_setup *s, enum gkyl_edge_loc edge, int upsample, int half_width, + double cutoff, struct gkyl_array *shift_dg, struct ts_ctx *tctx) { - double y = xn[1]; + struct gkyl_array *f = ts_donor_new(s); - double mu = 0.0; - double sigma = 0.3; + struct gkyl_bc_twistshift_inp inp = { + .bc_dir = ts_bc_dir, + .shift_dir = 1, + .shear_dir = 0, + .edge = edge, + .cdim = ts_cdim, + .bcdir_ext_update_r = &s->update_r, + .num_ghost = s->ghost, + .basis = &s->basis, + .grid = &s->grid, + .use_gpu = false, + .upsample_factor = upsample, + .filter_half_width = half_width, + .filter_cutoff_wavelength = cutoff, + }; + if (shift_dg) + inp.shift_dg = shift_dg; + else { + inp.shift_func = shift_func; + inp.shift_func_ctx = tctx; + } - fout[0] = ( 1.0/sqrt(2.0*M_PI*pow(sigma,2)) ) * exp( -pow(y-mu,2)/(2.0*pow(sigma,2)) ); + struct gkyl_bc_twistshift *up = gkyl_bc_twistshift_inew(&inp); + gkyl_bc_twistshift_advance(up, f, f); + gkyl_bc_twistshift_release(up); + return f; } -void -shift_fig9(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) +// Largest difference between two fields over a range. +static double +ts_max_diff(const struct gkyl_array *fa, const struct gkyl_array *fb, + const struct gkyl_range *rng, int num_basis) { - struct test_bc_twistshift_ctx *pars = ctx; - double Lx[2] = {pars->upper[0]-pars->lower[0], pars->upper[1]-pars->lower[1]}; - double dx[2] = {Lx[0]/pars->cells[0], Lx[1]/pars->cells[1]}; - - fout[0] = dx[1]/2.0; + double maxd = 0.0; + struct gkyl_range_iter iter; + gkyl_range_iter_init(&iter, rng); + while (gkyl_range_iter_next(&iter)) { + long linidx = gkyl_range_idx(rng, iter.idx); + const double *a = gkyl_array_cfetch(fa, linidx); + const double *b = gkyl_array_cfetch(fb, linidx); + for (int k=0; ksize; i++) { + const double *a = gkyl_array_cfetch(fa, i); + const double *b = gkyl_array_cfetch(fb, i); + for (int k=0; kbasis.num_basis; k++) + maxd = GKYL_MAX2(maxd, fabs(a[k]-b[k])); + } + return maxd; } -void -init_donor_fig9(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) +// Integral of the field over the plane the BC fills. +static double +ts_ghost_sum(const struct ts_setup *s, const struct gkyl_array *f) { - double y = xn[1]; - - struct test_bc_twistshift_ctx *pars = ctx; - double ymid = 0.5*(pars->upper[1]+pars->lower[1]); - double dy = (pars->upper[1]-pars->lower[1])/pars->cells[1]; - - fout[0] = 0.; - if (ymid < y && y < ymid+dy) - fout[0] = 1.; + double tot = 0.0; + struct gkyl_range_iter iter; + gkyl_range_iter_init(&iter, &s->ghost_r); + while (gkyl_range_iter_next(&iter)) + tot += ((const double *) gkyl_array_cfetch(f, gkyl_range_idx(&s->ghost_r, iter.idx)))[0]; + return tot; } void -test_bc_twistshift_3x_fig6_wcells(const int *cells, enum gkyl_edge_loc edge, - bool check_distf, bool use_gpu, bool write_f) -{ - double vt = 1.0; // Thermal speed. - double mass = 1.0; - double B0 = 1.0; // Magnetic field magnitude. - int bc_dir = 2; // Direction in which to apply TS. - - int poly_order = 1; - const double lower[] = {-2.0, -1.50, -3.0}; - const double upper[] = { 2.0, 1.50, 3.0}; - int vdim = 0; - int ndim = sizeof(lower)/sizeof(lower[0]); - int cdim = ndim - vdim; - - double lower_conf[cdim], upper_conf[cdim]; - int cells_conf[cdim]; - for (int d=0; dgeo_corn.bmag, 0.0); - gkyl_array_shiftc(gk_geom->geo_corn.bmag, B0*pow(sqrt(2.0),cdim), 0); - - struct gkyl_dg_updater_moment *mcalc = gkyl_dg_updater_moment_gyrokinetic_new(&grid, &basis_conf, - &basis, &local_conf, mass, 0, gvm, gk_geom, NULL, GKYL_F_MOMENT_M0M1M2, true, use_gpu); - int num_mom = gkyl_dg_updater_moment_gyrokinetic_num_mom(mcalc); - - struct gkyl_array *marr = mkarr(use_gpu, num_mom, local_ext_conf.volume); - double *red_integ_mom_skin, *red_integ_mom_ghost; - if (use_gpu) { - red_integ_mom_skin = gkyl_cu_malloc(sizeof(double[num_mom])); - red_integ_mom_ghost = gkyl_cu_malloc(sizeof(double[num_mom])); - } - else { - red_integ_mom_skin = gkyl_malloc(sizeof(double[num_mom])); - red_integ_mom_ghost = gkyl_malloc(sizeof(double[num_mom])); - } - double *red_integ_mom_skin_ho = gkyl_malloc(sizeof(double[num_mom])); - double *red_integ_mom_ghost_ho = gkyl_malloc(sizeof(double[num_mom])); - - gkyl_dg_updater_moment_gyrokinetic_advance(mcalc, - &skin_rng, &skin_rng_conf, distf, marr); - gkyl_array_reduce_range(red_integ_mom_skin, marr, GKYL_SUM, &skin_rng_conf); - - gkyl_dg_updater_moment_gyrokinetic_advance(mcalc, - &ghost_rng, &ghost_rng_conf, distf, marr); - gkyl_array_reduce_range(red_integ_mom_ghost, marr, GKYL_SUM, &ghost_rng_conf); - - if (use_gpu) { - gkyl_cu_memcpy(red_integ_mom_skin_ho, red_integ_mom_skin, sizeof(double[num_mom]), GKYL_CU_MEMCPY_D2H); - gkyl_cu_memcpy(red_integ_mom_ghost_ho, red_integ_mom_ghost, sizeof(double[num_mom]), GKYL_CU_MEMCPY_D2H); + gkyl_array_release(f_pre); } - else { - memcpy(red_integ_mom_skin_ho, red_integ_mom_skin, sizeof(double[num_mom])); - memcpy(red_integ_mom_ghost_ho, red_integ_mom_ghost, sizeof(double[num_mom])); - } - - for (int k=0; kupper[0]-pars->lower[0], pars->upper[1]-pars->lower[1]}; - double dx[2] = {Lx[0]/pars->cells[0], Lx[1]/pars->cells[1]}; - - fout[0] = 0.6*x+1.8; } void -init_donor_3x_fig11(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) +test_identity_filter_matches_plain(void) { - double x = xn[0], y = xn[1], z = xn[2]; - - struct test_bc_twistshift_ctx *pars = ctx; - double Lx[2] = {pars->upper[0]-pars->lower[0], pars->upper[1]-pars->lower[1]}; - double B0 = pars->B0; - double vt = pars->vt; - double mass = pars->mass; - double vtsq = vt*vt; - - double beta[2] = {0.0, 0.0}; - double sigma[2] = {0.6, 0.2}; + // A cutoff at the Nyquist wavelength makes the kernel the identity, so + // filtering must leave the plain twist-shift result alone. + struct ts_ctx tctx = { .offset = 0.75, .shear = 0.25 }; + double dx = (ts_upper[0]-ts_lower[0])/ts_cells[0]; - fout[0] = ( 1.0/pow(sqrt(2.0*M_PI*vtsq),3) ) - * exp( -pow(x-beta[0],2)/(2.0*pow(sigma[0],2)) -pow(y-beta[1],2)/(2.0*pow(sigma[1],2)) ); -} + struct ts_setup s; + ts_setup_init(&s, GKYL_LOWER_EDGE); -void -init_donor_3x2v_fig11(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) -{ - double x = xn[0], y = xn[1], z = xn[2], vpar = xn[3], mu = xn[4]; + struct gkyl_array *f_plain = ts_run(&s, GKYL_LOWER_EDGE, 1, 0, 0.0, NULL, &tctx); + struct gkyl_array *f_filt = ts_run(&s, GKYL_LOWER_EDGE, 1, 4, 2.0*dx, NULL, &tctx); - struct test_bc_twistshift_ctx *pars = ctx; - double B0 = pars->B0; - double vt = pars->vt; - double mass = pars->mass; - double vtsq = vt*vt; + double maxd = ts_max_diff(f_plain, f_filt, &s.ghost_r, s.basis.num_basis); + TEST_CHECK( maxd < 1.0e-13 ); + TEST_MSG("identity filter changed the result by %.3e", maxd); - init_donor_3x_fig11(t, xn, fout, ctx); - fout[0] *= exp( -(pow(vpar,2)+2.0*mu*B0/mass)/(2.0*vtsq) ); + gkyl_array_release(f_plain); + gkyl_array_release(f_filt); } void -test_bc_twistshift_3x_fig11_wcells(const int *cells, enum gkyl_edge_loc edge, - int apply_in_half_x, bool check_distf, bool use_gpu, bool write_f) +test_upsample_no_shear_matches_plain(void) { - double vt = 1.0; // Thermal speed. - double mass = 1.0; - double B0 = 1.0; // Magnetic field magnitude. - int bc_dir = 2; // Direction in which to apply TS. - - int poly_order = 1; - const double lower[] = {-2.0, -1.50, -3.0}; - const double upper[] = { 2.0, 1.50, 3.0}; - int vdim = 0; - int ndim = sizeof(lower)/sizeof(lower[0]); - int cdim = ndim - vdim; - - double lower_conf[cdim], upper_conf[cdim]; - int cells_conf[cdim]; - for (int d=0; d 0) { - // Apply the BC only on the upper half of the domain. - int x_half_len = (update_rng.upper[0] - update_rng.lower[0] + 1)/2; - gkyl_range_shorten_from_below(&update_rng, &update_rng, 0, x_half_len); - } - - // Create the twist-shift updater and shift the donor field. - struct gkyl_bc_twistshift_inp tsinp = { - .bc_dir = bc_dir, - .shift_dir = 1, // y shift. - .shear_dir = 0, // shift varies with x. - .edge = edge, - .cdim = cdim, - .bcdir_ext_update_r = &update_rng, - .num_ghost = ghost, - .basis = &basis, - .grid = &grid, - .shift_func = shift_fig11, - .shift_func_ctx = &proj_ctx, - .use_gpu = use_gpu, - }; - - struct gkyl_bc_twistshift *tsup = gkyl_bc_twistshift_inew(&tsinp); - - // First apply periodicity in z. - struct gkyl_array *buff_per = mkarr(use_gpu, basis.num_basis, skin_rng.volume); - apply_periodic_bc(buff_per, distf, bc_dir, skin_ghost); - - gkyl_bc_twistshift_advance(tsup, distf, distf); - gkyl_array_copy(distf_ho, distf); - - // Write out the target in the extended range. - if (write_f) { - double lower_ext[ndim], upper_ext[ndim]; - int cells_ext[ndim]; - for (int d=0; d 0) { - // Applied the BC only on the upper half of the domain. - int x_half_len = (ghost_rng.upper[0] - ghost_rng.lower[0] + 1)/2; - gkyl_range_shorten_from_below(&check_ghost_rng, &ghost_rng, 0, x_half_len); - gkyl_range_shorten_from_above(&check_other_ghost_rng, &ghost_rng, 0, x_half_len); - } - else - check_ghost_rng = ghost_rng; + struct gkyl_array *f_plain = ts_run(&s, GKYL_LOWER_EDGE, 1, 0, 0.0, NULL, &tctx); + struct gkyl_array *f_up = ts_run(&s, GKYL_LOWER_EDGE, upsample, 4, 2.0*dx/upsample, NULL, &tctx); - struct gkyl_range_iter iter; - gkyl_range_iter_init(&iter, &check_ghost_rng); - while (gkyl_range_iter_next(&iter)) { - long linidx = gkyl_range_idx(&check_ghost_rng, iter.idx); - double *f_c = gkyl_array_fetch(distf_ho, linidx); - int refidx = (iter.idx[0]-1)*cells[1] + iter.idx[1]-1; - TEST_CHECK( gkyl_compare(f0[refidx], f_c[0], 1e-13) ); - TEST_CHECK( gkyl_compare(f1[refidx], f_c[1], 1e-13) ); - TEST_CHECK( gkyl_compare(f2[refidx], f_c[2], 1e-12) ); - TEST_CHECK( gkyl_compare(f6[refidx], f_c[6], 1e-12) ); - } + double maxd = ts_max_diff(f_plain, f_up, &s.ghost_r, s.basis.num_basis); + TEST_CHECK( maxd < 1.0e-12 ); + TEST_MSG("upsample %d round trip changed the result by %.3e", upsample, maxd); - if (apply_in_half_x != 0) { - // Check that the other half is untouched. - int skin_idx[GKYL_MAX_DIM]; - gkyl_range_iter_init(&iter, &check_other_ghost_rng); - while (gkyl_range_iter_next(&iter)) { - long linidx = gkyl_range_idx(&check_other_ghost_rng, iter.idx); - double *f_c = gkyl_array_fetch(distf_ho, linidx); - - for (int d=0; dncomp; k++) - TEST_CHECK( gkyl_compare(fskin_c[k], f_c[k], 1e-15) ); - } - } + gkyl_array_release(f_plain); + gkyl_array_release(f_up); } - - gkyl_array_release(buff_per); - test_bc_twistshift_array_meta_release(mt); - gkyl_bc_twistshift_release(tsup); - gkyl_proj_on_basis_release(projDistf); - gkyl_array_release(distf_ho); - gkyl_array_release(distf); - } void -init_donor_3x_fig14(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) +test_shift_dg_matches_shift_func(void) { - double x = xn[0], y = xn[1], z = xn[2]; + // A discretized shift given on the field's own grid must give the same answer + // as the shift function, including when the shift has to be refined. + struct ts_ctx tctx = { .offset = 0.75, .shear = 0.25 }; + double dx = (ts_upper[0]-ts_lower[0])/ts_cells[0]; - struct test_bc_twistshift_ctx *pars = ctx; - double Lx[3] = { - pars->upper[0]-pars->lower[0], - pars->upper[1]-pars->lower[1], - pars->upper[2]-pars->lower[2], - }; - - double f_amplitude = 1.0; - double f_floor = 1.0e-10; + for (int upsample=1; upsample<=2; upsample++) { + struct ts_setup s; + ts_setup_init(&s, GKYL_LOWER_EDGE); + struct gkyl_array *shift_dg = ts_shift_dg_new(&s, &tctx); - // Cube - double rx2 = pow(x-Lx[0]/2,2); - double ry2 = pow(y-Lx[1]/2,2); - double rz2 = pow(z-Lx[2]/2,2); - - if (rx2 < pow(Lx[0]/4,2) && ry2 < pow(Lx[1]/4,2)) - fout[0] = f_amplitude; - else - fout[0] = f_floor; -} + int half_width = upsample > 1? 4 : 0; + double cutoff = upsample > 1? 2.0*dx : 0.0; -void -shift_fig14(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) -{ - double x = xn[0]; + struct gkyl_array *f_func = ts_run(&s, GKYL_LOWER_EDGE, upsample, half_width, cutoff, NULL, &tctx); + struct gkyl_array *f_dg = ts_run(&s, GKYL_LOWER_EDGE, upsample, half_width, cutoff, shift_dg, &tctx); - struct test_bc_twistshift_ctx *pars = ctx; - double Lx[3] = { - pars->upper[0]-pars->lower[0], - pars->upper[1]-pars->lower[1], - pars->upper[2]-pars->lower[2], - }; - double dx[3] = { - Lx[0]/pars->cells[0], - Lx[1]/pars->cells[1], - Lx[2]/pars->cells[2], - }; - enum gkyl_edge_loc edge = pars->edge; + double maxd = ts_max_diff(f_func, f_dg, &s.ghost_r, s.basis.num_basis); + TEST_CHECK( maxd < 1.0e-12 ); + TEST_MSG("upsample %d: shift_dg differs from shift_func by %.3e", upsample, maxd); - fout[0] = -(x-0.5); - - if (edge == GKYL_UPPER_EDGE) - fout[0] *= -1.0; + gkyl_array_release(f_func); + gkyl_array_release(f_dg); + gkyl_array_release(shift_dg); + } } void -test_bc_twistshift_3x_fig14_wcells(const int *cells, enum gkyl_edge_loc edge, - int apply_in_half_x, bool check_distf, bool use_gpu, bool write_f) -{ - double vt = 1.0; // Thermal speed. - double mass = 1.0; - double B0 = 1.0; // Magnetic field magnitude. - int bc_dir = 2; // Direction in which to apply TS. - - int poly_order = 1; - const double lower[] = {0.0, 0.0, 0.0}; - const double upper[] = {1.0, 1.0, 1.0}; - int vdim = 0; - int ndim = sizeof(lower)/sizeof(lower[0]); - int cdim = ndim - vdim; - - double lower_conf[cdim], upper_conf[cdim]; - int cells_conf[cdim]; - for (int d=0; d 0) { - // Apply the BC only on the upper half of the domain. - int x_half_len = (update_rng.upper[0] - update_rng.lower[0] + 1)/2; - gkyl_range_shorten_from_below(&update_rng, &update_rng, 0, x_half_len); - } - - // Create the twist-shift updater and shift the donor field. - struct gkyl_bc_twistshift_inp tsinp = { - .bc_dir = bc_dir, - .shift_dir = 1, // y shift. - .shear_dir = 0, // shift varies with x. - .edge = edge, - .cdim = cdim, - .bcdir_ext_update_r = &update_rng, - .num_ghost = ghost, - .basis = &basis, - .grid = &grid, - .shift_func = shift_fig14, - .shift_func_ctx = &proj_ctx, - .use_gpu = use_gpu, - }; - - struct gkyl_bc_twistshift *tsup = gkyl_bc_twistshift_inew(&tsinp); - - // First apply periodicity in z. - struct gkyl_array *buff_per = mkarr(use_gpu, basis.num_basis, skin_rng.volume); - apply_periodic_bc(buff_per, distf, bc_dir, skin_ghost); - - gkyl_bc_twistshift_advance(tsup, distf, distf); - gkyl_array_copy(distf_ho, distf); - - // Write out the target in the extended range. - if (write_f) { - double lower_ext[ndim], upper_ext[ndim]; - int cells_ext[ndim]; - for (int d=0; d 0) { - // Applied the BC only on the upper half of the domain. - int x_half_len = (ghost_rng.upper[0] - ghost_rng.lower[0] + 1)/2; - gkyl_range_shorten_from_below(&check_ghost_rng, &ghost_rng, 0, x_half_len); - gkyl_range_shorten_from_above(&check_other_ghost_rng, &ghost_rng, 0, x_half_len); - } - else - check_ghost_rng = ghost_rng; - - struct gkyl_range_iter iter; - gkyl_range_iter_init(&iter, &check_ghost_rng); - while (gkyl_range_iter_next(&iter)) { - long linidx = gkyl_range_idx(&check_ghost_rng, iter.idx); - double *f_c = gkyl_array_fetch(distf_ho, linidx); - int refidx = (iter.idx[0]-1)*cells[1] + iter.idx[1]-1; - TEST_CHECK( gkyl_compare(f0[refidx], f_c[0], 1e-13) ); - TEST_CHECK( gkyl_compare(f1[refidx], f_c[1], 1e-13) ); - TEST_CHECK( gkyl_compare(f2[refidx], f_c[2], 1e-12) ); - TEST_CHECK( gkyl_compare(f6[refidx], f_c[6], 1e-12) ); - } - } + TEST_CHECK( tv[1] < tv[0] ); + TEST_MSG("total variation along x: plain %.6e, de-aliased %.6e", tv[0], tv[1]); - gkyl_array_release(buff_per); - test_bc_twistshift_array_meta_release(mt); - gkyl_bc_twistshift_release(tsup); - gkyl_proj_on_basis_release(projDistf); - gkyl_array_release(distf_ho); - gkyl_array_release(distf); - + gkyl_array_release(f_plain); + gkyl_array_release(f_deal); } void -test_bc_twistshift_3x2v_fig11_wcells(const int *cells, enum gkyl_edge_loc edge, - int apply_in_half_x, bool check_distf, bool use_gpu, bool write_f) +test_conserves_particles(void) { - double vt = 1.0; // Thermal speed. - double mass = 1.0; - double B0 = 1.0; // Magnetic field magnitude. - int bc_dir = 2; // Direction in which to apply TS. - - int poly_order = 1; - const double lower[] = {-2.0, -1.50, -3.0, -5.0*vt, 0.}; - const double upper[] = { 2.0, 1.50, 3.0, 5.0*vt, mass*(pow(5.0*vt,2))/(2.0*B0)}; - int vdim = 2; - int ndim = sizeof(lower)/sizeof(lower[0]); - int cdim = ndim - vdim; - - double lower_conf[cdim], upper_conf[cdim]; - int cells_conf[cdim]; - for (int d=0; d 0) { - // Apply the BC only on the upper half of the domain. - int x_half_len = (update_rng.upper[0] - update_rng.lower[0] + 1)/2; - gkyl_range_shorten_from_below(&update_rng, &update_rng, 0, x_half_len); - } - - // Create the twist-shift updater and shift the donor field. - struct gkyl_bc_twistshift_inp tsinp = { - .bc_dir = bc_dir, - .shift_dir = 1, // y shift. - .shear_dir = 0, // shift varies with x. - .edge = edge, - .cdim = cdim, - .bcdir_ext_update_r = &update_rng, - .num_ghost = ghost, - .basis = &basis, - .grid = &grid, - .shift_func = shift_fig11, - .shift_func_ctx = &proj_ctx, - .use_gpu = use_gpu, - }; - - struct gkyl_bc_twistshift *tsup = gkyl_bc_twistshift_inew(&tsinp); - - // First apply periodicity in z. - struct gkyl_array *buff_per = mkarr(use_gpu, basis.num_basis, skin_rng.volume); - apply_periodic_bc(buff_per, distf, bc_dir, skin_ghost); - - gkyl_bc_twistshift_advance(tsup, distf, distf); - gkyl_array_copy(distf_ho, distf); - - // Write out the target in the extended range. - if (write_f) { - double lower_ext[ndim], upper_ext[ndim]; - int cells_ext[ndim]; - for (int d=0; dgeo_corn.bmag, 0.0); - gkyl_array_shiftc(gk_geom->geo_corn.bmag, B0*pow(sqrt(2.0),cdim), 0); - - struct gkyl_dg_updater_moment *mcalc = gkyl_dg_updater_moment_gyrokinetic_new(&grid, &basis_conf, - &basis, &local_conf, mass, 0, gvm, gk_geom, NULL, GKYL_F_MOMENT_M0M1M2, true, use_gpu); - int num_mom = gkyl_dg_updater_moment_gyrokinetic_num_mom(mcalc); - - struct gkyl_array *marr = mkarr(use_gpu, num_mom, local_ext_conf.volume); - double *red_integ_mom_skin, *red_integ_mom_ghost; - if (use_gpu) { - red_integ_mom_skin = gkyl_cu_malloc(sizeof(double[num_mom])); - red_integ_mom_ghost = gkyl_cu_malloc(sizeof(double[num_mom])); - } - else { - red_integ_mom_skin = gkyl_malloc(sizeof(double[num_mom])); - red_integ_mom_ghost = gkyl_malloc(sizeof(double[num_mom])); - } - double *red_integ_mom_skin_ho = gkyl_malloc(sizeof(double[num_mom])); - double *red_integ_mom_ghost_ho = gkyl_malloc(sizeof(double[num_mom])); - - gkyl_dg_updater_moment_gyrokinetic_advance(mcalc, - &skin_rng, &skin_rng_conf, distf, marr); - gkyl_array_reduce_range(red_integ_mom_skin, marr, GKYL_SUM, &skin_rng_conf); - - gkyl_dg_updater_moment_gyrokinetic_advance(mcalc, - &ghost_rng, &ghost_rng_conf, distf, marr); - gkyl_array_reduce_range(red_integ_mom_ghost, marr, GKYL_SUM, &ghost_rng_conf); - - if (use_gpu) { - gkyl_cu_memcpy(red_integ_mom_skin_ho, red_integ_mom_skin, sizeof(double[num_mom]), GKYL_CU_MEMCPY_D2H); - gkyl_cu_memcpy(red_integ_mom_ghost_ho, red_integ_mom_ghost, sizeof(double[num_mom]), GKYL_CU_MEMCPY_D2H); - } - else { - memcpy(red_integ_mom_skin_ho, red_integ_mom_skin, sizeof(double[num_mom])); - memcpy(red_integ_mom_ghost_ho, red_integ_mom_ghost, sizeof(double[num_mom])); - } - - for (int k=0; k 0) { - // Applied the BC only on the upper half of the domain. - int x_half_len = (ghost_rng.upper[0] - ghost_rng.lower[0] + 1)/2; - gkyl_range_shorten_from_below(&check_ghost_rng, &ghost_rng, 0, x_half_len); - gkyl_range_shorten_from_above(&check_other_ghost_rng, &ghost_rng, 0, x_half_len); - } - else - check_ghost_rng = ghost_rng; + for (int c=0; c<4; c++) { + struct gkyl_array *f = ts_run(&s, edge, upsample[c], half_width[c], cutoff[c], NULL, &tctx); + double tot = ts_ghost_sum(&s, f); - struct gkyl_range_iter iter; - gkyl_range_iter_init(&iter, &check_ghost_rng); - while (gkyl_range_iter_next(&iter)) { - if (iter.idx[3]==1 && iter.idx[4]==1) { - long linidx = gkyl_range_idx(&check_ghost_rng, iter.idx); - double *f_c = gkyl_array_fetch(distf_ho, linidx); - int refidx = (iter.idx[0]-1)*cells[1] + iter.idx[1]-1; - TEST_CHECK( gkyl_compare(f0[refidx], f_c[0], 1e-13) ); - TEST_CHECK( gkyl_compare(f1[refidx], f_c[1], 1e-13) ); - TEST_CHECK( gkyl_compare(f2[refidx], f_c[2], 1e-12) ); - TEST_CHECK( gkyl_compare(f6[refidx], f_c[6], 1e-12) ); - } - } + TEST_CHECK( fabs(tot-tot_pre) < 1.0e-12*fabs(tot_pre) ); + TEST_MSG("edge %d, config %d (upsample %d, M %d): %.13e -> %.13e, rel change %.3e", + e, c, upsample[c], half_width[c], tot_pre, tot, fabs(tot-tot_pre)/fabs(tot_pre)); - if (apply_in_half_x != 0) { - // Check that the other half is untouched. - int skin_idx[GKYL_MAX_DIM]; - gkyl_range_iter_init(&iter, &check_other_ghost_rng); - while (gkyl_range_iter_next(&iter)) { - long linidx = gkyl_range_idx(&check_other_ghost_rng, iter.idx); - double *f_c = gkyl_array_fetch(distf_ho, linidx); - - for (int d=0; dncomp; k++) - TEST_CHECK( gkyl_compare(fskin_c[k], f_c[k], 1e-15) ); - } + gkyl_array_release(f); } + gkyl_array_release(f_pre); } - - gkyl_free(red_integ_mom_skin_ho); - gkyl_free(red_integ_mom_ghost_ho); - if (use_gpu) { - gkyl_cu_free(red_integ_mom_skin); - gkyl_cu_free(red_integ_mom_ghost); - } - else { - gkyl_free(red_integ_mom_skin); - gkyl_free(red_integ_mom_ghost); - } - gkyl_dg_updater_moment_gyrokinetic_release(mcalc); - gkyl_array_release(marr); - gkyl_gk_geometry_release(gk_geom); - gkyl_position_map_release(pmap); - gkyl_velocity_map_release(gvm); - gkyl_array_release(buff_per); - test_bc_twistshift_array_meta_release(mt); - gkyl_bc_twistshift_release(tsup); - gkyl_proj_on_basis_release(projDistf); - gkyl_array_release(distf_ho); - gkyl_array_release(distf); - -} - -// CBC geometry (see rt_gk_cbc_passive_3x2v_p1.c) - -static double -interp_1x_lut_cbc(double x, double *lut_grid, double *lut_val, int N) -{ - double x_min = lut_grid[0]; - double x_max = lut_grid[N-1]; - if (x <= x_min) return lut_val[0]; - if (x >= x_max) return lut_val[N-1]; - double dx = (x_max - x_min)/(N-1); - int idx = (int)((x - x_min)/dx); - if (idx < 0) idx = 0; - if (idx >= N-1) idx = N-2; - return lut_val[idx] + (lut_val[idx+1] - lut_val[idx])*(x - lut_grid[idx])/(lut_grid[idx+1] - lut_grid[idx]); -} - -struct gk_cbc_app_ctx { - double a_shift, Z_axis, R_axis, R0, a_mid, r0, B0, kappa, delta, q0, Cy, qaxis, qlcfs; - double Lx, Ly, Lz; - double x_min, y_min, z_min, x_max, y_max, z_max; - int psi_lut_size; - double *r_lut; - double *dPsidr_int_lut; -}; - -struct integrand_cbc_ctx { - struct gk_cbc_app_ctx *app_ctx; - double r; - double theta; -}; - -static double r_x_cbc(double x, double r0) { return x + r0; } - -static double qprofile_cbc(double r, double a_mid, double qaxis, double qlcfs) -{ - return 1.0 + 2.78*pow(r/a_mid, 2.8); -} - -static double R_rtheta_cbc(double r, double theta, void *ctx) -{ - struct gk_cbc_app_ctx *app = ctx; - return app->R_axis - app->a_shift*r*r/(2.*app->R_axis) - + r*cos(theta + asin(app->delta)*sin(theta)); -} - -static double dRdr_cbc(double r, double theta, void *ctx) -{ - struct gk_cbc_app_ctx *app = ctx; - return -app->a_shift*r/app->R_axis + cos(theta + asin(app->delta)*sin(theta)); -} - -static double dRdtheta_cbc(double r, double theta, void *ctx) -{ - struct gk_cbc_app_ctx *app = ctx; - return -r*sin(theta + asin(app->delta)*sin(theta))*(1. + asin(app->delta)*cos(theta)); -} - -static double dZdr_cbc(double r, double theta, void *ctx) -{ - struct gk_cbc_app_ctx *app = ctx; - return app->kappa*sin(theta); -} - -static double dZdtheta_cbc(double r, double theta, void *ctx) -{ - struct gk_cbc_app_ctx *app = ctx; - return app->kappa*r*cos(theta); -} - -static double Jr_cbc(double r, double theta, void *ctx) -{ - return R_rtheta_cbc(r, theta, ctx) - *(dRdr_cbc(r, theta, ctx)*dZdtheta_cbc(r, theta, ctx) - - dRdtheta_cbc(r, theta, ctx)*dZdr_cbc(r, theta, ctx)); -} - -static double Bphi_cbc(double R, void *ctx) -{ - struct gk_cbc_app_ctx *app = ctx; - return app->B0*app->R0/R; -} - -static double integrand_JoRsq_cbc(double t, void *int_ctx) -{ - struct integrand_cbc_ctx *inctx = int_ctx; - return Jr_cbc(inctx->r, t, inctx->app_ctx) - / pow(R_rtheta_cbc(inctx->r, t, inctx->app_ctx), 2); -} - -static double intdPsidr_cbc(double r, void *ctx) -{ - struct gk_cbc_app_ctx *app = ctx; - struct integrand_cbc_ctx tmp_ctx = {.app_ctx = app, .r = r}; - struct gkyl_qr_res integral = gkyl_dbl_exp(integrand_JoRsq_cbc, &tmp_ctx, 0., 2.*M_PI, 7, 1e-10); - return integral.res; -} - -static double dPsidr_cbc(double r, double theta, void *ctx) -{ - struct gk_cbc_app_ctx *app = ctx; - double integral_val = interp_1x_lut_cbc(r, app->r_lut, app->dPsidr_int_lut, app->psi_lut_size); - double R = R_rtheta_cbc(r, theta, ctx); - double Bt = Bphi_cbc(R, ctx); - return (R*Bt/(2.*M_PI*qprofile_cbc(r, app->a_mid, app->qaxis, app->qlcfs)))*integral_val; -} - -static double compute_alpha_integral_cbc(double r, double twrap, void *ctx) -{ - struct gk_cbc_app_ctx *app = ctx; - struct integrand_cbc_ctx tmp_ctx = {.app_ctx = app, .r = r}; - struct gkyl_qr_res integral; - if (twrap == 0.0) return 0.0; - if (0. < twrap) { - integral = gkyl_dbl_exp(integrand_JoRsq_cbc, &tmp_ctx, 0., twrap, 7, 1e-10); - return integral.res; - } else { - integral = gkyl_dbl_exp(integrand_JoRsq_cbc, &tmp_ctx, twrap, 0., 7, 1e-10); - return -integral.res; - } -} - -static double alpha_cbc(double r, double theta, double phi, void *ctx) -{ - double twrap = theta; - while (twrap < -M_PI) twrap += 2.*M_PI; - while (twrap > M_PI) twrap -= 2.*M_PI; - double integral_val = compute_alpha_integral_cbc(r, twrap, ctx); - double R = R_rtheta_cbc(r, theta, ctx); - double Bt = Bphi_cbc(R, ctx); - return phi - R*Bt*integral_val/dPsidr_cbc(r, theta, ctx); -} - -void bc_shift_func_lo_cbc(double t, const double *xc, double* GKYL_RESTRICT fout, void *ctx) -{ - struct gk_cbc_app_ctx *app = ctx; - double r = r_x_cbc(xc[0], app->r0); - fout[0] = app->Cy*(alpha_cbc(r, app->z_min, 0.0, ctx) - alpha_cbc(r, app->z_max, 0.0, ctx)); -} - -void bc_shift_func_up_cbc(double t, const double *xc, double* GKYL_RESTRICT fout, void *ctx) -{ - struct gk_cbc_app_ctx *app = ctx; - double r = r_x_cbc(xc[0], app->r0); - fout[0] = -app->Cy*(alpha_cbc(r, app->z_min, 0.0, ctx) - alpha_cbc(r, app->z_max, 0.0, ctx)); -} - -void init_donor_3x_cbc(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) -{ - struct gk_cbc_app_ctx *app = ctx; - double x = xn[0], y = xn[1], z = xn[2]; - double sigx = app->Lx/7.0, sigy = app->Ly/7.0, sigz = app->Lz/10.0; - double mux = 0.*app->Lx; - fout[0] = exp(-pow(x-mux,2)/(2.*pow(sigx,2)) - pow(y,2)/(2.*pow(sigy,2)) - pow(z,2)/(2.*pow(sigz,2))); } void -test_bc_twistshift_3x_cbc_wcells(const int *cells, enum gkyl_edge_loc edge, - bool check_distf, bool use_gpu, bool write_f) +test_zero_shift_is_identity(void) { - int bc_dir = 2; - int poly_order = 1; - - // Physical parameters matching rt_gk_cbc_passive_3x2v_p1.c create_ctx(). - double eV = GKYL_ELEMENTARY_CHARGE; - double mp = GKYL_PROTON_MASS; - double qi = eV; - double mi = mp; // AMU = 1 (hydrogen) - double Te0 = 2000.*eV; - - double R_axis = 1.6714; - double B_axis = 1.54; - double a_mid = 0.604; - double R0 = R_axis + 0.5*a_mid; - double r0 = 0.5*a_mid; - double B0 = B_axis*(R_axis/R0); - double qaxis = 1.2; - double qlcfs = 2.0; - - double c_s = sqrt(Te0/mi); - double omega_ci = fabs(qi*B0/mi); - double rho_s = c_s/omega_ci; - double q0 = qprofile_cbc(r0, a_mid, qaxis, qlcfs); - double Cy = r0/q0; - - double Lx = 150.*rho_s; - double Ly = 150.*rho_s; - Ly = 2.*M_PI*Cy/round(2.*M_PI*Cy/Ly); // adjust to integer toroidal mode number - double Lz = 2.*M_PI - 1e-10; - - // Use a fixed LUT size sufficient for accurate geometry evaluation. - int psi_lut_size = 200; - - struct gk_cbc_app_ctx app_ctx = { - .a_shift = 0.0, .Z_axis = 0.0, - .R_axis = R_axis, .R0 = R0, .a_mid = a_mid, .r0 = r0, - .B0 = B0, .kappa = 1.0, .delta = 0.0, - .q0 = q0, .Cy = Cy, .qaxis = qaxis, .qlcfs = qlcfs, - .Lx = Lx, .Ly = Ly, .Lz = Lz, - .x_min = -Lx/2., .x_max = Lx/2., - .y_min = -Ly/2., .y_max = Ly/2., - .z_min = -Lz/2., .z_max = Lz/2., - .psi_lut_size = psi_lut_size, - }; + // Check that a zero shift leaves the ghost plane alone, for any upsampling. + struct ts_ctx tctx = { .offset = 0.0, .shear = 0.0 }; + double dx = (ts_upper[0]-ts_lower[0])/ts_cells[0]; - // Populate lookup tables (avoids redundant integration in geometry evaluations). - app_ctx.r_lut = gkyl_malloc(psi_lut_size*sizeof(double)); - app_ctx.dPsidr_int_lut = gkyl_malloc(psi_lut_size*sizeof(double)); - double r_lut_min = 0.0, r_lut_max = 2.0*a_mid; - for (int i=0; i 1? 4 : 0; + double cutoff = upsample[c] > 1? 2.0*dx/upsample[c] : 0.0; + struct gkyl_array *f = ts_run(&s, edge, upsample[c], half_width, cutoff, NULL, &tctx); - int ghost_conf[cdim]; - for (int d=0; d + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Compare two DG coefficients with a mixed relative/absolute tolerance. +// Needed to pass the CBC 3x test on stellar nvcc installation. +static bool +ts_compare_coeff(double ref, double val) +{ + return gkyl_compare(ref, val, 1e-13) || fabs(ref-val) < 1e-14; +} + +// Meta-data for IO +struct test_bc_twistshift_output_meta { + int poly_order; // polynomial order + const char *basis_type; // name of basis functions +}; + +// returned gkyl_array_meta must be freed using gyrokinetic_array_meta_release +static struct gkyl_msgpack_data* +test_bc_twistshift_array_meta_new(struct test_bc_twistshift_output_meta meta) +{ + struct gkyl_msgpack_data *mt = gkyl_malloc(sizeof(*mt)); + + mt->meta_sz = 0; + mpack_writer_t writer; + mpack_writer_init_growable(&writer, &mt->meta, &mt->meta_sz); + + // add some data to mpack + mpack_build_map(&writer); + + mpack_write_cstr(&writer, "polyOrder"); + mpack_write_i64(&writer, meta.poly_order); + + mpack_write_cstr(&writer, "basisType"); + mpack_write_cstr(&writer, meta.basis_type); + + mpack_complete_map(&writer); + + int status = mpack_writer_destroy(&writer); + + if (status != mpack_ok) { + free(mt->meta); // we need to use free here as mpack does its own malloc + gkyl_free(mt); + mt = 0; + } + + return mt; +} + +static void +test_bc_twistshift_array_meta_release(struct gkyl_msgpack_data *mt) +{ + if (!mt) return; + MPACK_FREE(mt->meta); + gkyl_free(mt); +} + +struct skin_ghost_ranges { + struct gkyl_range lower_skin[GKYL_MAX_DIM]; + struct gkyl_range lower_ghost[GKYL_MAX_DIM]; + + struct gkyl_range upper_skin[GKYL_MAX_DIM]; + struct gkyl_range upper_ghost[GKYL_MAX_DIM]; +}; + +// Create ghost and skin sub-ranges given a parent range +static void +skin_ghost_ranges_init(struct skin_ghost_ranges *sgr, + const struct gkyl_range *parent, const int *ghost) +{ + int ndim = parent->ndim; + + for (int d=0; dlower_skin[d], &sgr->lower_ghost[d], + d, GKYL_LOWER_EDGE, parent, ghost); + gkyl_skin_ghost_ranges(&sgr->upper_skin[d], &sgr->upper_ghost[d], + d, GKYL_UPPER_EDGE, parent, ghost); + } +} +// Apply periodic BCs along parallel direction +void +apply_periodic_bc(struct gkyl_array *buff, struct gkyl_array *fld, const int dir, const struct skin_ghost_ranges sgr) +{ + gkyl_array_copy_to_buffer(buff->data, fld, &(sgr.lower_skin[dir])); + gkyl_array_copy_from_buffer(fld, buff->data, &(sgr.upper_ghost[dir])); + + gkyl_array_copy_to_buffer(buff->data, fld, &(sgr.upper_skin[dir])); + gkyl_array_copy_from_buffer(fld, buff->data, &(sgr.lower_ghost[dir])); +} + +static struct gkyl_array* +mkarr(bool on_gpu, long nc, long size) +{ + struct gkyl_array* a; + if (on_gpu) + a = gkyl_array_cu_dev_new(GKYL_DOUBLE, nc, size); + else + a = gkyl_array_new(GKYL_DOUBLE, nc, size); + return a; +} + +struct test_bc_twistshift_ctx { + double lower[GKYL_MAX_DIM], upper[GKYL_MAX_DIM]; + int cells[GKYL_MAX_DIM]; + double B0; + double vt; + double mass; + enum gkyl_edge_loc edge; +}; + +void +mapc2p(double t, const double *xc, double* GKYL_RESTRICT xp, void *ctx) +{ + xp[0] = xc[0]; xp[1] = xc[1]; xp[2] = xc[2]; +} + +void eval_bfield_3x(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) +{ + double x = xn[0], y = xn[1], z = xn[2]; + + struct test_bc_twistshift_ctx *pars = ctx; + double B0 = pars->B0; + + fout[0] = 0.0; + fout[1] = 0.0; + fout[2] = B0; +} + +void +shift1_fig6(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) +{ + struct test_bc_twistshift_ctx *pars = ctx; + double Lx[2] = {pars->upper[0]-pars->lower[0], pars->upper[1]-pars->lower[1]}; + double dx[2] = {Lx[0]/pars->cells[0], Lx[1]/pars->cells[1]}; + + fout[0] = 4.0*dx[1]; +} + +void +shift1m_fig6(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) +{ + shift1_fig6(t, xn, fout, ctx); + fout[0] *= -1.0; +} + +void +shift2_fig6(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) +{ + fout[0] = 1.1; +} + +void +shift2m_fig6(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) +{ + shift2_fig6(t, xn, fout, ctx); + fout[0] *= -1.0; +} + +void +init_donor_fig6(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) +{ + double y = xn[1]; + + double mu = 0.0; + double sigma = 0.3; + + fout[0] = ( 1.0/sqrt(2.0*M_PI*pow(sigma,2)) ) * exp( -pow(y-mu,2)/(2.0*pow(sigma,2)) ); +} + +void +shift_fig9(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) +{ + struct test_bc_twistshift_ctx *pars = ctx; + double Lx[2] = {pars->upper[0]-pars->lower[0], pars->upper[1]-pars->lower[1]}; + double dx[2] = {Lx[0]/pars->cells[0], Lx[1]/pars->cells[1]}; + + fout[0] = dx[1]/2.0; +} + +void +shiftm_fig9(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) +{ + shift_fig9(t, xn, fout, ctx); + fout[0] *= -1.0; +} + +void +init_donor_fig9(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) +{ + double y = xn[1]; + + struct test_bc_twistshift_ctx *pars = ctx; + double ymid = 0.5*(pars->upper[1]+pars->lower[1]); + double dy = (pars->upper[1]-pars->lower[1])/pars->cells[1]; + + fout[0] = 0.; + if (ymid < y && y < ymid+dy) + fout[0] = 1.; +} + +void +test_bc_twistshift_3x_fig6_wcells(const int *cells, enum gkyl_edge_loc edge, + bool check_distf, bool use_gpu, bool write_f) +{ + double vt = 1.0; // Thermal speed. + double mass = 1.0; + double B0 = 1.0; // Magnetic field magnitude. + int bc_dir = 2; // Direction in which to apply TS. + + int poly_order = 1; + const double lower[] = {-2.0, -1.50, -3.0}; + const double upper[] = { 2.0, 1.50, 3.0}; + int vdim = 0; + int ndim = sizeof(lower)/sizeof(lower[0]); + int cdim = ndim - vdim; + + double lower_conf[cdim], upper_conf[cdim]; + int cells_conf[cdim]; + for (int d=0; dgeo_corn.bmag, 0.0); + gkyl_array_shiftc(gk_geom->geo_corn.bmag, B0*pow(sqrt(2.0),cdim), 0); + + struct gkyl_dg_updater_moment *mcalc = gkyl_dg_updater_moment_gyrokinetic_new(&grid, &basis_conf, + &basis, &local_conf, mass, 0, gvm, gk_geom, NULL, GKYL_F_MOMENT_M0M1M2, true, use_gpu); + int num_mom = gkyl_dg_updater_moment_gyrokinetic_num_mom(mcalc); + + struct gkyl_array *marr = mkarr(use_gpu, num_mom, local_ext_conf.volume); + double *red_integ_mom_skin, *red_integ_mom_ghost; + if (use_gpu) { + red_integ_mom_skin = gkyl_cu_malloc(sizeof(double[num_mom])); + red_integ_mom_ghost = gkyl_cu_malloc(sizeof(double[num_mom])); + } + else { + red_integ_mom_skin = gkyl_malloc(sizeof(double[num_mom])); + red_integ_mom_ghost = gkyl_malloc(sizeof(double[num_mom])); + } + double *red_integ_mom_skin_ho = gkyl_malloc(sizeof(double[num_mom])); + double *red_integ_mom_ghost_ho = gkyl_malloc(sizeof(double[num_mom])); + + gkyl_dg_updater_moment_gyrokinetic_advance(mcalc, + &skin_rng, &skin_rng_conf, distf, marr); + gkyl_array_reduce_range(red_integ_mom_skin, marr, GKYL_SUM, &skin_rng_conf); + + gkyl_dg_updater_moment_gyrokinetic_advance(mcalc, + &ghost_rng, &ghost_rng_conf, distf, marr); + gkyl_array_reduce_range(red_integ_mom_ghost, marr, GKYL_SUM, &ghost_rng_conf); + + if (use_gpu) { + gkyl_cu_memcpy(red_integ_mom_skin_ho, red_integ_mom_skin, sizeof(double[num_mom]), GKYL_CU_MEMCPY_D2H); + gkyl_cu_memcpy(red_integ_mom_ghost_ho, red_integ_mom_ghost, sizeof(double[num_mom]), GKYL_CU_MEMCPY_D2H); + } + else { + memcpy(red_integ_mom_skin_ho, red_integ_mom_skin, sizeof(double[num_mom])); + memcpy(red_integ_mom_ghost_ho, red_integ_mom_ghost, sizeof(double[num_mom])); + } + + for (int k=0; kupper[0]-pars->lower[0], pars->upper[1]-pars->lower[1]}; + double dx[2] = {Lx[0]/pars->cells[0], Lx[1]/pars->cells[1]}; + + fout[0] = 0.6*x+1.8; +} + +void +init_donor_3x_fig11(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) +{ + double x = xn[0], y = xn[1], z = xn[2]; + + struct test_bc_twistshift_ctx *pars = ctx; + double Lx[2] = {pars->upper[0]-pars->lower[0], pars->upper[1]-pars->lower[1]}; + double B0 = pars->B0; + double vt = pars->vt; + double mass = pars->mass; + double vtsq = vt*vt; + + double beta[2] = {0.0, 0.0}; + double sigma[2] = {0.6, 0.2}; + + fout[0] = ( 1.0/pow(sqrt(2.0*M_PI*vtsq),3) ) + * exp( -pow(x-beta[0],2)/(2.0*pow(sigma[0],2)) -pow(y-beta[1],2)/(2.0*pow(sigma[1],2)) ); +} + +void +init_donor_3x2v_fig11(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) +{ + double x = xn[0], y = xn[1], z = xn[2], vpar = xn[3], mu = xn[4]; + + struct test_bc_twistshift_ctx *pars = ctx; + double B0 = pars->B0; + double vt = pars->vt; + double mass = pars->mass; + double vtsq = vt*vt; + + init_donor_3x_fig11(t, xn, fout, ctx); + fout[0] *= exp( -(pow(vpar,2)+2.0*mu*B0/mass)/(2.0*vtsq) ); +} + +void +test_bc_twistshift_3x_fig11_wcells(const int *cells, enum gkyl_edge_loc edge, + int apply_in_half_x, bool check_distf, bool use_gpu, bool write_f) +{ + double vt = 1.0; // Thermal speed. + double mass = 1.0; + double B0 = 1.0; // Magnetic field magnitude. + int bc_dir = 2; // Direction in which to apply TS. + + int poly_order = 1; + const double lower[] = {-2.0, -1.50, -3.0}; + const double upper[] = { 2.0, 1.50, 3.0}; + int vdim = 0; + int ndim = sizeof(lower)/sizeof(lower[0]); + int cdim = ndim - vdim; + + double lower_conf[cdim], upper_conf[cdim]; + int cells_conf[cdim]; + for (int d=0; d 0) { + // Apply the BC only on the upper half of the domain. + int x_half_len = (update_rng.upper[0] - update_rng.lower[0] + 1)/2; + gkyl_range_shorten_from_below(&update_rng, &update_rng, 0, x_half_len); + } + + // Create the twist-shift updater and shift the donor field. + struct gkyl_twistshift_dg_inp tsinp = { + .bc_dir = bc_dir, + .shift_dir = 1, // y shift. + .shear_dir = 0, // shift varies with x. + .edge = edge, + .cdim = cdim, + .bcdir_ext_update_r = &update_rng, + .num_ghost = ghost, + .basis = &basis, + .grid = &grid, + .shift_func = shift_fig11, + .shift_func_ctx = &proj_ctx, + .use_gpu = use_gpu, + }; + + struct gkyl_twistshift_dg *tsup = gkyl_twistshift_dg_inew(&tsinp); + + // First apply periodicity in z. + struct gkyl_array *buff_per = mkarr(use_gpu, basis.num_basis, skin_rng.volume); + apply_periodic_bc(buff_per, distf, bc_dir, skin_ghost); + + gkyl_twistshift_dg_advance(tsup, distf, distf); + gkyl_array_copy(distf_ho, distf); + + // Write out the target in the extended range. + if (write_f) { + double lower_ext[ndim], upper_ext[ndim]; + int cells_ext[ndim]; + for (int d=0; d 0) { + // Applied the BC only on the upper half of the domain. + int x_half_len = (ghost_rng.upper[0] - ghost_rng.lower[0] + 1)/2; + gkyl_range_shorten_from_below(&check_ghost_rng, &ghost_rng, 0, x_half_len); + gkyl_range_shorten_from_above(&check_other_ghost_rng, &ghost_rng, 0, x_half_len); + } + else + check_ghost_rng = ghost_rng; + + struct gkyl_range_iter iter; + gkyl_range_iter_init(&iter, &check_ghost_rng); + while (gkyl_range_iter_next(&iter)) { + long linidx = gkyl_range_idx(&check_ghost_rng, iter.idx); + double *f_c = gkyl_array_fetch(distf_ho, linidx); + int refidx = (iter.idx[0]-1)*cells[1] + iter.idx[1]-1; + TEST_CHECK( gkyl_compare(f0[refidx], f_c[0], 1e-13) ); + TEST_CHECK( gkyl_compare(f1[refidx], f_c[1], 1e-13) ); + TEST_CHECK( gkyl_compare(f2[refidx], f_c[2], 1e-12) ); + TEST_CHECK( gkyl_compare(f6[refidx], f_c[6], 1e-12) ); + } + + if (apply_in_half_x != 0) { + // Check that the other half is untouched. + int skin_idx[GKYL_MAX_DIM]; + gkyl_range_iter_init(&iter, &check_other_ghost_rng); + while (gkyl_range_iter_next(&iter)) { + long linidx = gkyl_range_idx(&check_other_ghost_rng, iter.idx); + double *f_c = gkyl_array_fetch(distf_ho, linidx); + + for (int d=0; dncomp; k++) + TEST_CHECK( gkyl_compare(fskin_c[k], f_c[k], 1e-15) ); + } + } + } + + gkyl_array_release(buff_per); + test_bc_twistshift_array_meta_release(mt); + gkyl_twistshift_dg_release(tsup); + gkyl_proj_on_basis_release(projDistf); + gkyl_array_release(distf_ho); + gkyl_array_release(distf); + +} + +void +init_donor_3x_fig14(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) +{ + double x = xn[0], y = xn[1], z = xn[2]; + + struct test_bc_twistshift_ctx *pars = ctx; + double Lx[3] = { + pars->upper[0]-pars->lower[0], + pars->upper[1]-pars->lower[1], + pars->upper[2]-pars->lower[2], + }; + + double f_amplitude = 1.0; + double f_floor = 1.0e-10; + + // Cube + double rx2 = pow(x-Lx[0]/2,2); + double ry2 = pow(y-Lx[1]/2,2); + double rz2 = pow(z-Lx[2]/2,2); + + if (rx2 < pow(Lx[0]/4,2) && ry2 < pow(Lx[1]/4,2)) + fout[0] = f_amplitude; + else + fout[0] = f_floor; +} + +void +shift_fig14(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) +{ + double x = xn[0]; + + struct test_bc_twistshift_ctx *pars = ctx; + double Lx[3] = { + pars->upper[0]-pars->lower[0], + pars->upper[1]-pars->lower[1], + pars->upper[2]-pars->lower[2], + }; + double dx[3] = { + Lx[0]/pars->cells[0], + Lx[1]/pars->cells[1], + Lx[2]/pars->cells[2], + }; + enum gkyl_edge_loc edge = pars->edge; + + fout[0] = -(x-0.5); + + if (edge == GKYL_UPPER_EDGE) + fout[0] *= -1.0; +} + +void +test_bc_twistshift_3x_fig14_wcells(const int *cells, enum gkyl_edge_loc edge, + int apply_in_half_x, bool check_distf, bool use_gpu, bool write_f) +{ + double vt = 1.0; // Thermal speed. + double mass = 1.0; + double B0 = 1.0; // Magnetic field magnitude. + int bc_dir = 2; // Direction in which to apply TS. + + int poly_order = 1; + const double lower[] = {0.0, 0.0, 0.0}; + const double upper[] = {1.0, 1.0, 1.0}; + int vdim = 0; + int ndim = sizeof(lower)/sizeof(lower[0]); + int cdim = ndim - vdim; + + double lower_conf[cdim], upper_conf[cdim]; + int cells_conf[cdim]; + for (int d=0; d 0) { + // Apply the BC only on the upper half of the domain. + int x_half_len = (update_rng.upper[0] - update_rng.lower[0] + 1)/2; + gkyl_range_shorten_from_below(&update_rng, &update_rng, 0, x_half_len); + } + + // Create the twist-shift updater and shift the donor field. + struct gkyl_twistshift_dg_inp tsinp = { + .bc_dir = bc_dir, + .shift_dir = 1, // y shift. + .shear_dir = 0, // shift varies with x. + .edge = edge, + .cdim = cdim, + .bcdir_ext_update_r = &update_rng, + .num_ghost = ghost, + .basis = &basis, + .grid = &grid, + .shift_func = shift_fig14, + .shift_func_ctx = &proj_ctx, + .use_gpu = use_gpu, + }; + + struct gkyl_twistshift_dg *tsup = gkyl_twistshift_dg_inew(&tsinp); + + // First apply periodicity in z. + struct gkyl_array *buff_per = mkarr(use_gpu, basis.num_basis, skin_rng.volume); + apply_periodic_bc(buff_per, distf, bc_dir, skin_ghost); + + gkyl_twistshift_dg_advance(tsup, distf, distf); + gkyl_array_copy(distf_ho, distf); + + // Write out the target in the extended range. + if (write_f) { + double lower_ext[ndim], upper_ext[ndim]; + int cells_ext[ndim]; + for (int d=0; d 0) { + // Applied the BC only on the upper half of the domain. + int x_half_len = (ghost_rng.upper[0] - ghost_rng.lower[0] + 1)/2; + gkyl_range_shorten_from_below(&check_ghost_rng, &ghost_rng, 0, x_half_len); + gkyl_range_shorten_from_above(&check_other_ghost_rng, &ghost_rng, 0, x_half_len); + } + else + check_ghost_rng = ghost_rng; + + struct gkyl_range_iter iter; + gkyl_range_iter_init(&iter, &check_ghost_rng); + while (gkyl_range_iter_next(&iter)) { + long linidx = gkyl_range_idx(&check_ghost_rng, iter.idx); + double *f_c = gkyl_array_fetch(distf_ho, linidx); + int refidx = (iter.idx[0]-1)*cells[1] + iter.idx[1]-1; + TEST_CHECK( gkyl_compare(f0[refidx], f_c[0], 1e-13) ); + TEST_CHECK( gkyl_compare(f1[refidx], f_c[1], 1e-13) ); + TEST_CHECK( gkyl_compare(f2[refidx], f_c[2], 1e-12) ); + TEST_CHECK( gkyl_compare(f6[refidx], f_c[6], 1e-12) ); + } + + } + + gkyl_array_release(buff_per); + test_bc_twistshift_array_meta_release(mt); + gkyl_twistshift_dg_release(tsup); + gkyl_proj_on_basis_release(projDistf); + gkyl_array_release(distf_ho); + gkyl_array_release(distf); + +} + +void +test_bc_twistshift_3x2v_fig11_wcells(const int *cells, enum gkyl_edge_loc edge, + int apply_in_half_x, bool check_distf, bool use_gpu, bool write_f) +{ + double vt = 1.0; // Thermal speed. + double mass = 1.0; + double B0 = 1.0; // Magnetic field magnitude. + int bc_dir = 2; // Direction in which to apply TS. + + int poly_order = 1; + const double lower[] = {-2.0, -1.50, -3.0, -5.0*vt, 0.}; + const double upper[] = { 2.0, 1.50, 3.0, 5.0*vt, mass*(pow(5.0*vt,2))/(2.0*B0)}; + int vdim = 2; + int ndim = sizeof(lower)/sizeof(lower[0]); + int cdim = ndim - vdim; + + double lower_conf[cdim], upper_conf[cdim]; + int cells_conf[cdim]; + for (int d=0; d 0) { + // Apply the BC only on the upper half of the domain. + int x_half_len = (update_rng.upper[0] - update_rng.lower[0] + 1)/2; + gkyl_range_shorten_from_below(&update_rng, &update_rng, 0, x_half_len); + } + + // Create the twist-shift updater and shift the donor field. + struct gkyl_twistshift_dg_inp tsinp = { + .bc_dir = bc_dir, + .shift_dir = 1, // y shift. + .shear_dir = 0, // shift varies with x. + .edge = edge, + .cdim = cdim, + .bcdir_ext_update_r = &update_rng, + .num_ghost = ghost, + .basis = &basis, + .grid = &grid, + .shift_func = shift_fig11, + .shift_func_ctx = &proj_ctx, + .use_gpu = use_gpu, + }; + + struct gkyl_twistshift_dg *tsup = gkyl_twistshift_dg_inew(&tsinp); + + // First apply periodicity in z. + struct gkyl_array *buff_per = mkarr(use_gpu, basis.num_basis, skin_rng.volume); + apply_periodic_bc(buff_per, distf, bc_dir, skin_ghost); + + gkyl_twistshift_dg_advance(tsup, distf, distf); + gkyl_array_copy(distf_ho, distf); + + // Write out the target in the extended range. + if (write_f) { + double lower_ext[ndim], upper_ext[ndim]; + int cells_ext[ndim]; + for (int d=0; dgeo_corn.bmag, 0.0); + gkyl_array_shiftc(gk_geom->geo_corn.bmag, B0*pow(sqrt(2.0),cdim), 0); + + struct gkyl_dg_updater_moment *mcalc = gkyl_dg_updater_moment_gyrokinetic_new(&grid, &basis_conf, + &basis, &local_conf, mass, 0, gvm, gk_geom, NULL, GKYL_F_MOMENT_M0M1M2, true, use_gpu); + int num_mom = gkyl_dg_updater_moment_gyrokinetic_num_mom(mcalc); + + struct gkyl_array *marr = mkarr(use_gpu, num_mom, local_ext_conf.volume); + double *red_integ_mom_skin, *red_integ_mom_ghost; + if (use_gpu) { + red_integ_mom_skin = gkyl_cu_malloc(sizeof(double[num_mom])); + red_integ_mom_ghost = gkyl_cu_malloc(sizeof(double[num_mom])); + } + else { + red_integ_mom_skin = gkyl_malloc(sizeof(double[num_mom])); + red_integ_mom_ghost = gkyl_malloc(sizeof(double[num_mom])); + } + double *red_integ_mom_skin_ho = gkyl_malloc(sizeof(double[num_mom])); + double *red_integ_mom_ghost_ho = gkyl_malloc(sizeof(double[num_mom])); + + gkyl_dg_updater_moment_gyrokinetic_advance(mcalc, + &skin_rng, &skin_rng_conf, distf, marr); + gkyl_array_reduce_range(red_integ_mom_skin, marr, GKYL_SUM, &skin_rng_conf); + + gkyl_dg_updater_moment_gyrokinetic_advance(mcalc, + &ghost_rng, &ghost_rng_conf, distf, marr); + gkyl_array_reduce_range(red_integ_mom_ghost, marr, GKYL_SUM, &ghost_rng_conf); + + if (use_gpu) { + gkyl_cu_memcpy(red_integ_mom_skin_ho, red_integ_mom_skin, sizeof(double[num_mom]), GKYL_CU_MEMCPY_D2H); + gkyl_cu_memcpy(red_integ_mom_ghost_ho, red_integ_mom_ghost, sizeof(double[num_mom]), GKYL_CU_MEMCPY_D2H); + } + else { + memcpy(red_integ_mom_skin_ho, red_integ_mom_skin, sizeof(double[num_mom])); + memcpy(red_integ_mom_ghost_ho, red_integ_mom_ghost, sizeof(double[num_mom])); + } + + for (int k=0; k 0) { + // Applied the BC only on the upper half of the domain. + int x_half_len = (ghost_rng.upper[0] - ghost_rng.lower[0] + 1)/2; + gkyl_range_shorten_from_below(&check_ghost_rng, &ghost_rng, 0, x_half_len); + gkyl_range_shorten_from_above(&check_other_ghost_rng, &ghost_rng, 0, x_half_len); + } + else + check_ghost_rng = ghost_rng; + + struct gkyl_range_iter iter; + gkyl_range_iter_init(&iter, &check_ghost_rng); + while (gkyl_range_iter_next(&iter)) { + if (iter.idx[3]==1 && iter.idx[4]==1) { + long linidx = gkyl_range_idx(&check_ghost_rng, iter.idx); + double *f_c = gkyl_array_fetch(distf_ho, linidx); + int refidx = (iter.idx[0]-1)*cells[1] + iter.idx[1]-1; + TEST_CHECK( gkyl_compare(f0[refidx], f_c[0], 1e-13) ); + TEST_CHECK( gkyl_compare(f1[refidx], f_c[1], 1e-13) ); + TEST_CHECK( gkyl_compare(f2[refidx], f_c[2], 1e-12) ); + TEST_CHECK( gkyl_compare(f6[refidx], f_c[6], 1e-12) ); + } + } + + if (apply_in_half_x != 0) { + // Check that the other half is untouched. + int skin_idx[GKYL_MAX_DIM]; + gkyl_range_iter_init(&iter, &check_other_ghost_rng); + while (gkyl_range_iter_next(&iter)) { + long linidx = gkyl_range_idx(&check_other_ghost_rng, iter.idx); + double *f_c = gkyl_array_fetch(distf_ho, linidx); + + for (int d=0; dncomp; k++) + TEST_CHECK( gkyl_compare(fskin_c[k], f_c[k], 1e-15) ); + } + } + } + + gkyl_free(red_integ_mom_skin_ho); + gkyl_free(red_integ_mom_ghost_ho); + if (use_gpu) { + gkyl_cu_free(red_integ_mom_skin); + gkyl_cu_free(red_integ_mom_ghost); + } + else { + gkyl_free(red_integ_mom_skin); + gkyl_free(red_integ_mom_ghost); + } + gkyl_dg_updater_moment_gyrokinetic_release(mcalc); + gkyl_array_release(marr); + gkyl_gk_geometry_release(gk_geom); + gkyl_position_map_release(pmap); + gkyl_velocity_map_release(gvm); + gkyl_array_release(buff_per); + test_bc_twistshift_array_meta_release(mt); + gkyl_twistshift_dg_release(tsup); + gkyl_proj_on_basis_release(projDistf); + gkyl_array_release(distf_ho); + gkyl_array_release(distf); + +} + +// CBC geometry (see rt_gk_cbc_passive_3x2v_p1.c) + +static double +interp_1x_lut_cbc(double x, double *lut_grid, double *lut_val, int N) +{ + double x_min = lut_grid[0]; + double x_max = lut_grid[N-1]; + if (x <= x_min) return lut_val[0]; + if (x >= x_max) return lut_val[N-1]; + double dx = (x_max - x_min)/(N-1); + int idx = (int)((x - x_min)/dx); + if (idx < 0) idx = 0; + if (idx >= N-1) idx = N-2; + return lut_val[idx] + (lut_val[idx+1] - lut_val[idx])*(x - lut_grid[idx])/(lut_grid[idx+1] - lut_grid[idx]); +} + +struct gk_cbc_app_ctx { + double a_shift, Z_axis, R_axis, R0, a_mid, r0, B0, kappa, delta, q0, Cy, qaxis, qlcfs; + double Lx, Ly, Lz; + double x_min, y_min, z_min, x_max, y_max, z_max; + int psi_lut_size; + double *r_lut; + double *dPsidr_int_lut; +}; + +struct integrand_cbc_ctx { + struct gk_cbc_app_ctx *app_ctx; + double r; + double theta; +}; + +static double r_x_cbc(double x, double r0) { return x + r0; } + +static double qprofile_cbc(double r, double a_mid, double qaxis, double qlcfs) +{ + return 1.0 + 2.78*pow(r/a_mid, 2.8); +} + +static double R_rtheta_cbc(double r, double theta, void *ctx) +{ + struct gk_cbc_app_ctx *app = ctx; + return app->R_axis - app->a_shift*r*r/(2.*app->R_axis) + + r*cos(theta + asin(app->delta)*sin(theta)); +} + +static double dRdr_cbc(double r, double theta, void *ctx) +{ + struct gk_cbc_app_ctx *app = ctx; + return -app->a_shift*r/app->R_axis + cos(theta + asin(app->delta)*sin(theta)); +} + +static double dRdtheta_cbc(double r, double theta, void *ctx) +{ + struct gk_cbc_app_ctx *app = ctx; + return -r*sin(theta + asin(app->delta)*sin(theta))*(1. + asin(app->delta)*cos(theta)); +} + +static double dZdr_cbc(double r, double theta, void *ctx) +{ + struct gk_cbc_app_ctx *app = ctx; + return app->kappa*sin(theta); +} + +static double dZdtheta_cbc(double r, double theta, void *ctx) +{ + struct gk_cbc_app_ctx *app = ctx; + return app->kappa*r*cos(theta); +} + +static double Jr_cbc(double r, double theta, void *ctx) +{ + return R_rtheta_cbc(r, theta, ctx) + *(dRdr_cbc(r, theta, ctx)*dZdtheta_cbc(r, theta, ctx) + - dRdtheta_cbc(r, theta, ctx)*dZdr_cbc(r, theta, ctx)); +} + +static double Bphi_cbc(double R, void *ctx) +{ + struct gk_cbc_app_ctx *app = ctx; + return app->B0*app->R0/R; +} + +static double integrand_JoRsq_cbc(double t, void *int_ctx) +{ + struct integrand_cbc_ctx *inctx = int_ctx; + return Jr_cbc(inctx->r, t, inctx->app_ctx) + / pow(R_rtheta_cbc(inctx->r, t, inctx->app_ctx), 2); +} + +static double intdPsidr_cbc(double r, void *ctx) +{ + struct gk_cbc_app_ctx *app = ctx; + struct integrand_cbc_ctx tmp_ctx = {.app_ctx = app, .r = r}; + struct gkyl_qr_res integral = gkyl_dbl_exp(integrand_JoRsq_cbc, &tmp_ctx, 0., 2.*M_PI, 7, 1e-10); + return integral.res; +} + +static double dPsidr_cbc(double r, double theta, void *ctx) +{ + struct gk_cbc_app_ctx *app = ctx; + double integral_val = interp_1x_lut_cbc(r, app->r_lut, app->dPsidr_int_lut, app->psi_lut_size); + double R = R_rtheta_cbc(r, theta, ctx); + double Bt = Bphi_cbc(R, ctx); + return (R*Bt/(2.*M_PI*qprofile_cbc(r, app->a_mid, app->qaxis, app->qlcfs)))*integral_val; +} + +static double compute_alpha_integral_cbc(double r, double twrap, void *ctx) +{ + struct gk_cbc_app_ctx *app = ctx; + struct integrand_cbc_ctx tmp_ctx = {.app_ctx = app, .r = r}; + struct gkyl_qr_res integral; + if (twrap == 0.0) return 0.0; + if (0. < twrap) { + integral = gkyl_dbl_exp(integrand_JoRsq_cbc, &tmp_ctx, 0., twrap, 7, 1e-10); + return integral.res; + } else { + integral = gkyl_dbl_exp(integrand_JoRsq_cbc, &tmp_ctx, twrap, 0., 7, 1e-10); + return -integral.res; + } +} + +static double alpha_cbc(double r, double theta, double phi, void *ctx) +{ + double twrap = theta; + while (twrap < -M_PI) twrap += 2.*M_PI; + while (twrap > M_PI) twrap -= 2.*M_PI; + double integral_val = compute_alpha_integral_cbc(r, twrap, ctx); + double R = R_rtheta_cbc(r, theta, ctx); + double Bt = Bphi_cbc(R, ctx); + return phi - R*Bt*integral_val/dPsidr_cbc(r, theta, ctx); +} + +void bc_shift_func_lo_cbc(double t, const double *xc, double* GKYL_RESTRICT fout, void *ctx) +{ + struct gk_cbc_app_ctx *app = ctx; + double r = r_x_cbc(xc[0], app->r0); + fout[0] = app->Cy*(alpha_cbc(r, app->z_min, 0.0, ctx) - alpha_cbc(r, app->z_max, 0.0, ctx)); +} + +void bc_shift_func_up_cbc(double t, const double *xc, double* GKYL_RESTRICT fout, void *ctx) +{ + struct gk_cbc_app_ctx *app = ctx; + double r = r_x_cbc(xc[0], app->r0); + fout[0] = -app->Cy*(alpha_cbc(r, app->z_min, 0.0, ctx) - alpha_cbc(r, app->z_max, 0.0, ctx)); +} + +void init_donor_3x_cbc(double t, const double *xn, double* GKYL_RESTRICT fout, void *ctx) +{ + struct gk_cbc_app_ctx *app = ctx; + double x = xn[0], y = xn[1], z = xn[2]; + double sigx = app->Lx/7.0, sigy = app->Ly/7.0, sigz = app->Lz/10.0; + double mux = 0.*app->Lx; + fout[0] = exp(-pow(x-mux,2)/(2.*pow(sigx,2)) - pow(y,2)/(2.*pow(sigy,2)) - pow(z,2)/(2.*pow(sigz,2))); +} + +void +test_bc_twistshift_3x_cbc_wcells(const int *cells, enum gkyl_edge_loc edge, + bool check_distf, bool use_gpu, bool write_f) +{ + int bc_dir = 2; + int poly_order = 1; + + // Physical parameters matching rt_gk_cbc_passive_3x2v_p1.c create_ctx(). + double eV = GKYL_ELEMENTARY_CHARGE; + double mp = GKYL_PROTON_MASS; + double qi = eV; + double mi = mp; // AMU = 1 (hydrogen) + double Te0 = 2000.*eV; + + double R_axis = 1.6714; + double B_axis = 1.54; + double a_mid = 0.604; + double R0 = R_axis + 0.5*a_mid; + double r0 = 0.5*a_mid; + double B0 = B_axis*(R_axis/R0); + double qaxis = 1.2; + double qlcfs = 2.0; + + double c_s = sqrt(Te0/mi); + double omega_ci = fabs(qi*B0/mi); + double rho_s = c_s/omega_ci; + double q0 = qprofile_cbc(r0, a_mid, qaxis, qlcfs); + double Cy = r0/q0; + + double Lx = 150.*rho_s; + double Ly = 150.*rho_s; + Ly = 2.*M_PI*Cy/round(2.*M_PI*Cy/Ly); // adjust to integer toroidal mode number + double Lz = 2.*M_PI - 1e-10; + + // Use a fixed LUT size sufficient for accurate geometry evaluation. + int psi_lut_size = 200; + + struct gk_cbc_app_ctx app_ctx = { + .a_shift = 0.0, .Z_axis = 0.0, + .R_axis = R_axis, .R0 = R0, .a_mid = a_mid, .r0 = r0, + .B0 = B0, .kappa = 1.0, .delta = 0.0, + .q0 = q0, .Cy = Cy, .qaxis = qaxis, .qlcfs = qlcfs, + .Lx = Lx, .Ly = Ly, .Lz = Lz, + .x_min = -Lx/2., .x_max = Lx/2., + .y_min = -Ly/2., .y_max = Ly/2., + .z_min = -Lz/2., .z_max = Lz/2., + .psi_lut_size = psi_lut_size, + }; + + // Populate lookup tables (avoids redundant integration in geometry evaluations). + app_ctx.r_lut = gkyl_malloc(psi_lut_size*sizeof(double)); + app_ctx.dPsidr_int_lut = gkyl_malloc(psi_lut_size*sizeof(double)); + double r_lut_min = 0.0, r_lut_max = 2.0*a_mid; + for (int i=0; i #include -#include +#include +#include #include #include -#include -#include -#include -#include - -// Notes: -// a) Hard-coded parameters: -// - wrap_to_range: eps. -// - find_donors: delta_frac, num_test_pt. -// - find_intersect: tol, max_iter, num_steps. -// - calc_mats: shift_dir_idx_tar. -// - tol_xi: Minimum allowed spacing between the lower and -// upper xi (logical x) limits of subcell integral. -// b) Unlike the procedures described in M. Francisquez, et al. CPC 298 -// (2024) 109109, all subcell integrals are now done with variable y limits. -// This is possible once we realize that figure 4 is not drawn accurately; -// the blue lines should be separated by Delta y at all points. -// c) This updater only works on 5D distributions. Likely only minor changes -// are needed to make it work in other dimensions. -// d) 99% of the code is written to support a BC, a shift and shear in any -// direction. Maybe the only thing that needs to change is the permutted -// range and its use. -// -// List of functions used in computing sub-cell integrals (scimat). -// - ts_grid_cell_boundary_in_dir: cell boundary coordinate in given dir. -// - ts_grid_cell_boundaries: get all cell boundary coords. -// - ts_p2l: physical to logical transform. -// - ts_interval_dx_and_xc: compute length and center of an interval. -// - ts_grid_length_in_dir: length of the grid in given dir. -// - ts_wrap_to_range: wrap a number to a range assuming periodicity. -// - ts_shift_dir_idx_do_linidx: linear index to first donor of a given target -// cell in shift_dir_idx_do. -// - ts_check_shifted_test_point: evaluate a shifted point's cell as a -// potential donor cell. -// - ts_find_donors: find and record the donor cells for each target. -// - ts_root_find: Finds the root of a given function. -// - ts_shifted_coord_loss_func: Loss function used to find where yTar-S -// intersects yDo. -// - ts_sign: return the sign of a double. -// - ts_ts_donor_target_offset: offset between donor and target cells. -// - ts_find_intersect: finds the intersection of yTar-S and yDo. -// - ts_comp_to_phys: transform a computational to a physical coord. -// - ts_nod2mod_proj_1d: evaluate a 1D function at nodes and do a n2m transform -// to get the coefficients of the DG representation. -// - ts_integral_xlimdg: subcell integral with variable x limits. -// - ts_integral_ylimdg: subcell integral with variable y limits. -// - ts_integral_fullcelllimdg: integral over the whole cell. -// - ts_one: return 1 (for projections). -// - ts_minus_one: return -1 (for projections). -// - ts_shift_coord_shifted_log: coordinate in shift_dir shifted and transformed -// to logical space. -// - ts_subcellint_sNi_sNii: subcell integral sNi or sNii. -// - ts_subcellint_si_sii: subcell integral si or sii. -// - ts_subcellint_siii_siv: subcell integral siii or siv. -// - ts_subcellint_sv_svi: subcell integral sv or svi. -// - ts_subcellint_svii_sviii: subcell integral svii or sviii. -// - ts_subcellint_six_sx: subcell integral six or sx. -// - ts_subcellint_sxi_sxii: subcell integral sxi or sxii. -// - ts_subcellint_sxiii_sxiv: subcell integral sxiii or sxiv. -// - ts_subcellint_sxv_sxvi: subcell integral sxv or sxvi. -// - ts_calc_mats: create scimat with the result of the subcell integrals. -// -// Two additional helper functions: -// - ts_calc_num_numcol_fidx_do: index map to populate fmat with donors. -// - ts_calc_num_numcol_fidx_tar: index map to get mat-mat mult results. - -// Option to use the user-provided function describing the shift -// or a DG representation of it: -// = 0 DG representation (default and preferred). -// = 1 user-provided shift function. -// Note: the kernels that ultimately perform the integrals -// always use the DG representation. -#define shift_func_op 0 - -// Minimum allowed spacing between the lower and -// upper xi (logical x) limits of subcell integral. -#define tol_xi 1.0e-15 - -// Indices in 4-element cell boundary array. -#define cellb_lo(dir) (2*dir) -#define cellb_up(dir) (2*dir+1) - -double -ts_grid_cell_boundary_in_dir(struct gkyl_rect_grid *grid, const int *idx, enum gkyl_edge_loc edge, int dir) -{ - // Get the coordinate of the cell boundary in specified direction. - double xc[grid->ndim]; - gkyl_rect_grid_cell_center(grid, idx, xc); - return edge == GKYL_LOWER_EDGE? xc[dir]-0.5*grid->dx[dir] : xc[dir]+0.5*grid->dx[dir]; -} - -void -ts_grid_cell_boundaries(struct gkyl_rect_grid *grid, const int *idx, double *cell_bounds) -{ - // Get the cell boundaries in every dimension. The array cell_bounds - // must be a 2*grid->ndim array. - for (int d=0; dndim; d++) { - cell_bounds[d*2] = ts_grid_cell_boundary_in_dir(grid, idx, GKYL_LOWER_EDGE, d); - cell_bounds[d*2+1] = ts_grid_cell_boundary_in_dir(grid, idx, GKYL_UPPER_EDGE, d); - } -} - -static inline double -ts_p2l(double coord, double cell_center, double dx) -{ - // Transform a physical coordinate (coord) to the [-1,1] logical - // space in a cell centered at cell_center and with length dx. - return 2.0*(coord - cell_center)/dx; -} - -// Evaluation of the shift through the DG representation. -static inline void -ts_shift_dg_eval(double t, const double *coord, double *fout, void *ctx) -{ - struct ts_shift_dg_eval_ctx *tsectx = ctx; - - int cell_idx[GKYL_MAX_DIM]; - gkyl_rect_grid_coord_idx(tsectx->shear_grid, coord, cell_idx); - // Ensure that we do not go outside of the range - // (it does sometimes if x=x_max,x_min). - cell_idx[0] = fmin(cell_idx[0], tsectx->shear_r->upper[0]); - cell_idx[0] = fmax(cell_idx[0], tsectx->shear_r->lower[0]); - - double xc[GKYL_MAX_DIM]; - gkyl_rect_grid_cell_center(tsectx->shear_grid, cell_idx, xc); - - long shift_loc = gkyl_range_idx(tsectx->shear_r, cell_idx); - double *shift_c = (double *) gkyl_array_fetch(tsectx->shift_dg, shift_loc); - double xp = ts_p2l(coord[0], xc[0], tsectx->shear_grid->dx[0]); - - fout[0] = tsectx->shift_b->eval_expand(&(double) {xp}, shift_c); -} - -void -ts_interval_dx_and_xc(const double *interval, double *dx, double *xc) -{ - // Compute the lenth (dx) and center (xc) of [interval[0], interval[1]]. - double lo = interval[0], up = interval[1]; - dx[0] = up - lo; - xc[0] = 0.5*(up + lo); -} - -static inline double -ts_grid_length_in_dir(struct gkyl_rect_grid *grid, int dir) -{ - return grid->upper[dir] - grid->lower[dir]; -} - -double -ts_wrap_to_range(double val, double lower, double upper, bool pick_upper) -{ - // Wrap a number to range [lower,upper]. If pickUpper=true, output upper when - // val is a multiple of upper. Otherwise multiples of upper wrap to lower. - double L = upper - lower; - double disp = fmod(val - lower, L); - double vwrapped = lower + fmod(L + disp, L); - double eps = 1.e-12; - if ( (lower-eps < vwrapped && vwrapped < lower + eps) || - (upper-eps < vwrapped && vwrapped < upper + eps) ) { - if (pick_upper) - return upper; - else - return lower; - } - else - return vwrapped; -} - -long -ts_shift_dir_idx_do_linidx(const int *num_do, int shear_dir_idx, int shift_dir_idx, - int shift_dir_num_cells, int shear_r_lower) -{ - // Return the linear index to the first donor for the idx=(i,j) target cell, - // in the shift_dir_idx_do array. We assume shift_dir_idx_do (whose dimensions - // are Nx,Ny,num_do(i)) is in row-major order, and that it has num_do donors - // at each cell in the shear_dir_in_ts_grid direction. - long linc = 0; - // Count the number of donors in cells with an idx in the shear dir lower - // than this one. NOTE: the -1 here is because the idx is often 1-index - // (since ghost cells are the 0th index) but num_do is only defined on the - // local range. - for (int i=0; i -void -ts_check_shifted_test_point(struct gkyl_bc_twistshift *up, const double *test_pt, const double *xc, - const double *dx, double *shift_c, const int *idx, bool pick_lower, int *num_do_curr, - gkyl_mem_buff shift_dir_idx_do_buff) +// allocate array (filled with zeros) +static inline struct gkyl_array* +mkarr(bool use_gpu, long nc, long size) { - // Shift the test point, find the cell that contains it, and if we - // haven't included it yet, add it to our list of donors. - - int shear_idx[] = {idx[up->shear_dir_in_ts_grid]}; - int shift_idx[] = {idx[up->shift_dir_in_ts_grid]}; - - int *shift_dir_idx_do_buff_ptr = (int *) gkyl_mem_buff_data(shift_dir_idx_do_buff); - - // Evaluate the shift at this test point. - double test_pt_in_shear_dir = test_pt[up->shear_dir_in_ts_grid]; - double xc_in_shear_dir = xc[up->shear_dir_in_ts_grid]; - double dx_in_shear_dir = dx[up->shear_dir_in_ts_grid]; - double shift_at_pt = up->shift_b.eval_expand( - &(double) {ts_p2l(test_pt_in_shear_dir, xc_in_shear_dir, dx_in_shear_dir)}, shift_c); - - // Find the index of the cell that owns the shifted point. - double shifted_test_pt[] = { ts_wrap_to_range(test_pt[up->shift_dir_in_ts_grid] - shift_at_pt, - up->ts_grid.lower[up->shift_dir_in_ts_grid], up->ts_grid.upper[up->shift_dir_in_ts_grid], - false) }; // Shifted test point. - int shift_dir_idx_test_pt[1]; - gkyl_rect_grid_find_cell(&up->shift_grid, shifted_test_pt, (bool[]) {pick_lower}, (int[]) {-1}, shift_dir_idx_test_pt); - - // Get the linear index to the list of donors for this target. - long linidx = ts_shift_dir_idx_do_linidx(up->num_do, - shear_idx[0], shift_idx[0], up->ts_grid.cells[up->shift_dir_in_ts_grid], up->shear_r.lower[0]); - // If this donor is not in our list of donors, include it. - bool donor_not_found = true; - for (int k=0; k 0) { - // Insert one more int. Only if num_do_curr>0 because we already - // allocated space for the first donor. - size_t new_buff_sz = gkyl_mem_buff_size(shift_dir_idx_do_buff) + sizeof(int); - shift_dir_idx_do_buff = gkyl_mem_buff_resize(shift_dir_idx_do_buff, new_buff_sz); - } - - // Get the pointer again in case it changed. - shift_dir_idx_do_buff_ptr = (int *) gkyl_mem_buff_data(shift_dir_idx_do_buff); - shift_dir_idx_do_buff_ptr[linidx+num_do_curr[0]] = shift_dir_idx_test_pt[0]; - - num_do_curr[0] += 1; - } + return use_gpu? gkyl_array_cu_dev_new(GKYL_DOUBLE, nc, size) + : gkyl_array_new(GKYL_DOUBLE, nc, size); } -void -ts_find_donors(struct gkyl_bc_twistshift *up) +static void +bc_twistshift_refine_enabled(struct gkyl_bc_twistshift *up, struct gkyl_array *fdo) { - // Find the donor cells for each target cell in the TS grid. - - double delta_frac = 1.e-4; // Distance away from the boundary, as fraction of cell length. - // Must be larger than the eps=1e-6 tolerance in is_in_cell (used by - // gkyl_rect_grid_find_cell) so that shifted test points are never - // ambiguously on a cell boundary. - int num_test_pt[2] = {10, 10}; // Number of test points taken along each side of the cell. - - double step_sz[2] = {0.0}; // Size of the step between test points. - double delta[2] = {0.0}; // Space between cell boundary and test points. - for (int d=0; d<2; d++) { - delta[d] = delta_frac*up->ts_grid.dx[d]; - step_sz[d] = (up->ts_grid.dx[d] - 2.0*delta[d])/(num_test_pt[d]-1); - } - - // Number of donors at each cell of the shear direction. - up->num_do = (int*) gkyl_malloc(up->shear_r.volume * sizeof(int)); - for (int i=0; ishear_r.volume; i++) - up->num_do[i] = -1; - - // Temporary buffer to store donors at (resized below). - size_t curr_buff_sz = up->ts_r.volume * sizeof(int); - gkyl_mem_buff shift_dir_idx_do_buff = gkyl_mem_buff_new(curr_buff_sz); - - int idx[] = {up->shear_r.lower[0]}; - long linidx = gkyl_range_idx(&up->shear_r, idx); - struct gkyl_range_iter iter; - gkyl_range_iter_init(&iter, &up->ts_r); - while (gkyl_range_iter_next(&iter)) { - - // Get the cell boundaries and cell center. - double cell_b[4] = {0.0}; // Cell boundaries, x lo and up, y lo and up; - double xc[2] = {0.0}; // Cell center. - ts_grid_cell_boundaries(&up->ts_grid, iter.idx, cell_b); - gkyl_rect_grid_cell_center(&up->ts_grid, iter.idx, xc); - - int shear_idx[] = {iter.idx[up->shear_dir_in_ts_grid]}; - int shift_idx[] = {iter.idx[up->shift_dir_in_ts_grid]}; - long shift_loc = gkyl_range_idx(&up->shear_r, shear_idx); - double *shift_c = (double *) gkyl_array_fetch(up->shift_dg, shift_loc); - - int num_do_curr = 0; - - for (int dC=0; dC<2; dC++) { // dC=0: x=const, dC=1: y=const (boundaries). - for (int xS=0; xS<2; xS++) { // xS=0: lower, xS=1 upper (boundary). - - double test_pt[2] = {0.0}; // Test point to shift. - for (int d=0; d<2; d++) - test_pt[d] = cell_b[2*d]+delta[d]; - - // Search first shifted point. Use pick_lower=false in find_cell unless - // searching for points along a x=const line near the upper y-boundary. - bool pick_lower = dC==0 && xS==1; - test_pt[dC] += xS*(up->ts_grid.dx[dC]-2.0*delta[dC]); - - // Shift the test point, find the cell that contains it, and if we - // haven't included it yet, add it to our list of donors. - ts_check_shifted_test_point(up, test_pt, xc, up->ts_grid.dx, - shift_c, iter.idx, pick_lower, &num_do_curr, shift_dir_idx_do_buff); - - // Search for other shifted points along this line. - int step_dim = (dC+1) % 2; - for (int sI=1; sIts_grid.dx, - shift_c, iter.idx, pick_lower, &num_do_curr, shift_dir_idx_do_buff); - } - } - } - - up->num_do[shear_idx[0]-up->shear_r.lower[0]] = num_do_curr; - - } - - // Copy the donor list to the persistent object and release the buffer. - size_t buff_sz = gkyl_mem_buff_size(shift_dir_idx_do_buff); - up->shift_dir_idx_do = (int *) gkyl_malloc(buff_sz); - int *shift_dir_idx_do_buff_ptr = (int *) gkyl_mem_buff_data(shift_dir_idx_do_buff); - memcpy(up->shift_dir_idx_do, shift_dir_idx_do_buff_ptr, buff_sz); - gkyl_mem_buff_release(shift_dir_idx_do_buff); + gkyl_dg_interpolate_advance(up->refine, fdo, up->ffine); } -struct gkyl_qr_res -ts_root_find(double (*func)(double,void*), void *ctx, const double *lims, int max_iter, double tol) +static void +bc_twistshift_refine_disabled(struct gkyl_bc_twistshift *up, struct gkyl_array *fdo) { - // Use a Ridder's root finder to find the root of func in the interval - // [lims[0],lims[1]] down to a tolerance 'tol'. Return the interval limit - // if the function is smaller than the tolerance there. Return nil if the - // function does not change sign in the interval (interval doesn't contain the root). - double funcLo = func(lims[0], ctx), funcUp = func(lims[1], ctx); -// if (fabs(funcLo) < tol) -// return (struct gkyl_qr_res) {.res=lims[0], .status=0, .nevals=2}; -// else if (fabs(funcUp) < tol) -// return (struct gkyl_qr_res) {.res=lims[1], .status=0, .nevals=2}; -// else { -// if (funcLo*funcUp < 0) -// return gkyl_ridders(func, ctx, lims[0], lims[1], funcLo, funcUp, max_iter, tol); -// else -// return (struct gkyl_qr_res) {.status=1, .nevals=2}; -// } - if (fabs(funcLo) > tol && fabs(funcUp) > tol) { - if (funcLo*funcUp < 0) - return gkyl_ridders(func, ctx, lims[0], lims[1], funcLo, funcUp, max_iter, tol); - else - return (struct gkyl_qr_res) {.status=1, .nevals=2}; - } - else if (fabs(funcLo) < tol && fabs(funcUp) < tol) - return (struct gkyl_qr_res) {.status=1, .nevals=2}; - else if (fabs(funcLo) < tol) - return (struct gkyl_qr_res) {.res=lims[0], .status=0, .nevals=2}; - else if (fabs(funcUp) < tol) - return (struct gkyl_qr_res) {.res=lims[1], .status=0, .nevals=2}; - return (struct gkyl_qr_res) {.status=1, .nevals=2}; + gkyl_array_copy_range_to_range(up->ffine, fdo, &up->ghost_r, &up->coarse_ghost_r); } -struct ts_shifted_coord_loss_func_ctx { - double shiftCoordTar; // Target coordinate in shift_dir. - double shiftCoordDo; // Donor coordinate in shift_dir. - double shiftDirL; // Length of the domain in shift_dir. - int periodicCopyIdx; // Used to search a periodic copy of the domain (signed). - evalf_t shift_func; // Function defining the shift. - void *shift_func_ctx; // Context for shift_func. -}; - -double ts_shifted_coord_loss_func(double shearCoord, void *ctx) +static void +bc_twistshift_coarsen_enabled(struct gkyl_bc_twistshift *up, struct gkyl_array *ftar) { - // Loss function used to find the shear coord. - struct ts_shifted_coord_loss_func_ctx *tsctx = ctx; - - double shift; - tsctx->shift_func(0.0, (double[]){shearCoord}, &shift, tsctx->shift_func_ctx); - - return tsctx->shiftCoordTar - shift - - (tsctx->shiftCoordDo - tsctx->periodicCopyIdx * tsctx->shiftDirL); + gkyl_dg_interpolate_advance(up->coarsen, up->ffine, ftar); } -int static inline -ts_sign(double a) +static void +bc_twistshift_coarsen_disabled(struct gkyl_bc_twistshift *up, struct gkyl_array *ftar) { - if (a < 0.0) - return -1; - else if (a > 0.0) - return 1; - else - return 0; -} - -double -ts_donor_target_offset(struct gkyl_bc_twistshift *up, const double *xc_do, const double *xc_tar) { - // y-offset between the donor and the target cell (yDo-yTar), in the direction of the shift. - // xc_do: cell center coordinates of donor cell. - // xc_tar: cell center coordinates of target cell. - int shear_dir = up->shear_dir_in_ts_grid; - int shift_dir = up->shift_dir_in_ts_grid; - double x_eval = xc_do[up->shear_dir]; - double shift; - up->shift_func(0.0, (double[]){x_eval}, &shift, up->shift_func_ctx); - - int shift_sign = ts_sign(shift); - double shift_dir_L = up->ts_grid.upper[up->shift_dir] - up->ts_grid.lower[up->shift_dir]; - - // The idea here is that we keep shifting the donor cell center until it is in a - // periodic copy of our domain which overlaps with the shifted target cell center. - double xs_shifted_do = xc_do[shift_dir]; - double xs_shifted_tar = xc_tar[shift_dir] - shift; - bool keep_shifting = true; - while (keep_shifting) { - double xs_shifted_dolo = xs_shifted_do - shift_dir_L/2.0; - double xs_shifted_doup = xs_shifted_do + shift_dir_L/2.0; - if (xs_shifted_dolo <= xs_shifted_tar && xs_shifted_tar <= xs_shifted_doup) { - keep_shifting = false; - break; - } - else - xs_shifted_do = xs_shifted_do - shift_sign*shift_dir_L; - } - return xc_tar[shift_dir] - xs_shifted_do; + gkyl_array_copy_range_to_range(ftar, up->ffine, &up->coarse_ghost_r, &up->ghost_r); } -struct gkyl_qr_res -ts_find_intersect(struct gkyl_bc_twistshift *up, double shiftCoordTar, double shiftCoordDo, - const double *shearDirBounds, const double *shiftDirLimits, int nP_primary) +static void +bc_twistshift_advance_ts(struct gkyl_bc_twistshift *up, struct gkyl_array *fdo, + struct gkyl_array *ftar) { - // Given a y-coordinate of the target cell (yTar), and a y-coordinate - // of the donor cell (yDo), find the x-coordinate where yTar-yShift(x)=yDo - // for the periodic copy identified by nP_primary, i.e. the root of - // yTar - yShift(x) - (yDo - nP_primary*Ly) = 0 in [shearDirBounds[0], shearDirBounds[1]]. - // Returns status=1 (not found) if no root exists for that specific nP_primary. - // - // nP_primary must be the same for all 4 inter_pts of a donor-target pair: all four - // intersection conditions describe corners of a single physical overlap diamond and - // therefore belong to the same periodic copy of the domain. - double tol = 1.e-13; - int max_iter = 100; - - double shiftDirL = shiftDirLimits[1] - shiftDirLimits[0]; - - struct ts_shifted_coord_loss_func_ctx func_ctx = { - .shiftCoordTar = shiftCoordTar, - .shiftCoordDo = shiftCoordDo, - .shiftDirL = shiftDirL, - .periodicCopyIdx = nP_primary, - .shift_func = up->shift_func, - .shift_func_ctx = up->shift_func_ctx, - }; - return ts_root_find(ts_shifted_coord_loss_func, &func_ctx, shearDirBounds, max_iter, tol); + gkyl_twistshift_dg_advance(up->ts, fdo, ftar); } -struct ts_val_found { - bool status; // =0 not found, =1 found. - double value; // value found. -}; - -static inline void -ts_comp_to_phys(int ndim, const double *eta, - const double * GKYL_RESTRICT dx, const double * GKYL_RESTRICT xc, - double* GKYL_RESTRICT xout) +static void +bc_twistshift_advance_ts_filtered(struct gkyl_bc_twistshift *up, struct gkyl_array *fdo, + struct gkyl_array *ftar) { - for (int d=0; drefine_func(up, fdo); + gkyl_twistshift_dg_advance(up->ts, up->ffine, up->ffine); + gkyl_dg_lowpass_filter_advance(up->filter, up->ffine, up->filt_buff); + gkyl_array_copy_range(up->ffine, up->filt_buff, &up->ghost_r); + up->coarsen_func(up, ftar); } -void -ts_nod2mod_proj_1d(struct gkyl_bc_twistshift *up, evalf_t func, void *func_ctx, const double *interval, double *out) +static void +bc_twistshift_refine_shift(const struct gkyl_bc_twistshift_inp *inp, + const struct gkyl_rect_grid *ts_grid, const struct gkyl_range *shear_r_fine, + struct gkyl_array *shift_dg_fine) { - // Project 'func' onto 1D DG basis in [interval[0], interval[1]] - // evaluating at nodes and transforming to modal representation. - // func: 1D scalar function to be projected. - // interval: limits of the interval in which to project the function. - // out: DG field output. - double dx[] = {0.0}, xc[] = {0.0}, xmu[] = {0.0}; - ts_interval_dx_and_xc(interval, dx, xc); + // Refine the DG shift to match the supersampled shear grid. + int shear_dir = inp->shear_dir; + int shift_poly_order = inp->shift_poly_order? inp->shift_poly_order : inp->basis->poly_order; - for (int i=0; ishift_b.num_basis; ++i) { - ts_comp_to_phys(1, gkyl_eval_on_nodes_fetch_node(up->ev_on_nod1d, i), - dx, xc, xmu); - func(0.0, xmu, (double *)gkyl_array_fetch(up->func_nod1d,i), func_ctx); - } - gkyl_eval_on_nodes_nod2mod(up->ev_on_nod1d, up->func_nod1d, out); -} - -void -ts_integral_xlimdg(struct gkyl_bc_twistshift *up, double sFac, const double *xLimLo, - const double *xLimUp, double yLimLo, double yLimUp, double dyDo, double yOff, - const double *ySh, struct gkyl_mat *mat_do) { - // Populate a matrix (mat_do) with a sub-cell integral that has variably x limits - // represented by a DG polynomial, and a y-integral that goes from yLimLo to yLimUp. - // up: BC updater. - // sFac: +/-1 factor to add or subtract this subcell integral. - // xLimLo: DG representation of the lower x-limit. - // xLimUp: DG representation of the upper x-limit. - // yLimLo: lower y-limit. - // yLimUp: upper y-limit. - // dyDo: Cell length along y. - // yOff: Offset along y. - // ySh: DG representation of the y shift. - // mat_do: donor matrix. - up->kernels->xlimdg(sFac, xLimLo, xLimUp, yLimLo, yLimUp, dyDo, yOff, ySh, mat_do); -} - -void -ts_integral_ylimdg(struct gkyl_bc_twistshift *up, double sFac, double xLimLo, double xLimUp, - const double *yLimLo, const double *yLimUp, double dyDo, double yOff, - const double *ySh, struct gkyl_mat *mat_do) { - // Populate a matrix (mat_do) with a sub-cell integral that has variable y limits - // represented by a DG polynomial, and a x-integral that goes from xLimLo to xLimUp. - // up: BC updater. - // sFac: +/-1 factor to add or subtract this subcell integral. - // xLimLo: lower x-limit. - // xLimUp: upper x-limit. - // yLimLo: DG representation of the lower y-limit. - // yLimUp: DG representation of the upper y-limit. - // dyDo: Cell length along y. - // yOff: Offset along y. - // ySh: DG representation of the y shift. - // mat_do: donor matrix. - up->kernels->ylimdg(sFac, xLimLo, xLimUp, yLimLo, yLimUp, dyDo, yOff, ySh, mat_do); -} + struct gkyl_basis shift_b; + gkyl_cart_modal_serendip(&shift_b, 1, shift_poly_order); + assert(inp->shift_dg->ncomp == shift_b.num_basis); + assert(shift_dg_fine->ncomp == shift_b.num_basis); + assert(shift_dg_fine->size == shear_r_fine->volume); -void -ts_integral_fullcelllimdg(struct gkyl_bc_twistshift *up, double dyDo, double yOff, - const double *ySh, struct gkyl_mat *mat_do) { - // Populate a matrix (mat_do) with the full-cell integral. - // up: BC updater. - // sFac: +/-1 factor to add or subtract this subcell integral. - // dyDo: Cell length along y. - // yOff: Offset along y. - // ySh: DG representation of the y shift. - // mat_do: donor matrix. - up->kernels->fullcell(dyDo, yOff, ySh, mat_do); -} + struct gkyl_rect_grid grid_do, grid_tar; + gkyl_rect_grid_init(&grid_do, 1, &inp->grid->lower[shear_dir], + &inp->grid->upper[shear_dir], &inp->grid->cells[shear_dir]); + gkyl_rect_grid_init(&grid_tar, 1, &ts_grid->lower[shear_dir], + &ts_grid->upper[shear_dir], &ts_grid->cells[shear_dir]); -static inline void -ts_one(double t, const double *xn, double *fout, void *ctx) -{ - fout[0] = 1.0; -} + struct gkyl_range shear_r_do; + gkyl_range_init(&shear_r_do, 1, (int[]) {inp->bcdir_ext_update_r->lower[shear_dir]}, + (int[]) {inp->bcdir_ext_update_r->upper[shear_dir]}); + assert(inp->shift_dg->size == shear_r_do.volume); -static inline void -ts_minus_one(double t, const double *xn, double *fout, void *ctx) -{ - fout[0] = -1.0; -} - -struct ts_shift_coord_shifted_log_ctx { - double shift_coord_tar; // Target coordinate in shift_dir. - int shift_sign_fac; - const double *xc_do, *xc_tar; // Cell centers (donor and target). - double *dx; // Cell lengths. - bool pick_upper; - int shear_dir, shift_dir; // Shear and shift directions. - double shift_dir_bounds[2]; // Domain boundaries in shift_dir. - evalf_t shift_func; // Function defining the shift. - void *shift_func_ctx; // Context for shift_func. -}; - -void -ts_shift_coord_shifted_log(double t, const double *xn, double *fout, void *ctx) -{ - // Given a logical space x coordinate (xi) and a (physical) y-coordinate in the target cell, - // compute the shifted y-coordinate in the logical space of the donor cell (eta \in [-1,1]). - // xi: logical space x coordinate. - // yTar: physical y-coordinate in target cell. - // pmSh: factor multiplying the y-shift (+/- 1). - // xcDo: cell center coordinates of donor cell. - // xcTar: cell center coordinates of target cell. - // dx: cell lengths. - // pickUpper: boolean indicating if wrapping function should return upper/lower boundary. - - double xi = xn[0]; - - struct ts_shift_coord_shifted_log_ctx *tsctx = ctx; - double shift_coord_tar = tsctx->shift_coord_tar; - int shift_sign_fac = tsctx->shift_sign_fac; - const double *xc_do = tsctx->xc_do, *xc_tar = tsctx->xc_tar; - double *dx = tsctx->dx; - bool pick_upper = tsctx->pick_upper; - int shear_dir = tsctx->shear_dir, shift_dir = tsctx->shift_dir; - double *shift_dir_bounds = tsctx->shift_dir_bounds; - - double shear_coord_phys = xc_tar[shear_dir] + 0.5*dx[shear_dir]*xi; - double shift; - tsctx->shift_func(0.0, (double[]){shear_coord_phys}, &shift, tsctx->shift_func_ctx); - - double shift_coord_shifted = shift_coord_tar - shift_sign_fac * shift; - shift_coord_shifted = ts_wrap_to_range(shift_coord_shifted, shift_dir_bounds[0], shift_dir_bounds[1], pick_upper); - - fout[0] = ts_p2l(shift_coord_shifted, xc_do[shift_dir], dx[shift_dir]); -} - -void -ts_subcellint_sNi_sNii(struct gkyl_bc_twistshift *up, struct ts_val_found *inter_pts, - const double *xc_do, const double *xc_tar, const double *cellb_do, const double *cellb_tar, - bool is_upper_shift_dir_cell, const double *shift_c, struct gkyl_mat *mat_do) -{ - // Perform sNi or sNii subcell integrals. - // inter_pts: intersections y_{j_tar-/+1/2}-yShift and y_{j_do-/+1/2} (lower/upper y-boundaries of donor cell). - // xc_do: cell center of donor cell. - // xc_tar: cell center of target cell. - // cellb_do: boundaries of target cell. - // cellb_tar: boundaries of target cell. - // is_upper_shift_dir_cell: is the donor the upper cell in the shift dir? - // shift_c: DG coefficients of the shift. - // mat_do: current donor matrix. - - bool is_sNi = inter_pts[2].value < inter_pts[0].value; - - struct ts_shift_coord_shifted_log_ctx eta_lims_ctx = { - .shift_sign_fac = 1, - .xc_do = xc_do, - .xc_tar = xc_tar, - .dx = up->ts_grid.dx, - .pick_upper = is_upper_shift_dir_cell, - .shear_dir = up->shear_dir_in_ts_grid, - .shift_dir = up->shift_dir_in_ts_grid, - .shift_dir_bounds = {up->ts_grid.lower[up->shift_dir_in_ts_grid], - up->ts_grid.upper[up->shift_dir_in_ts_grid]}, - .shift_func = up->shift_func, - .shift_func_ctx = up->shift_func_ctx, - }; - - double xi_b[2]; // Limits of xi integral. - evalf_t eta_lims[2]; // Table of functions definting the limits of eta integral. - double etalo_xi[up->shift_b.num_basis], etaup_xi[up->shift_b.num_basis]; - - // Offset between cell centers in direction of the shift. - double xs_off = ts_donor_target_offset(up, xc_do, xc_tar); - - if (is_sNi) { - // sNi - // 1) Add the contribution of the left portion. - xi_b[0] = ts_p2l(inter_pts[2].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - xi_b[1] = ts_p2l(inter_pts[3].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - - eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_up(up->shift_dir_in_ts_grid)]; - eta_lims[0] = ts_minus_one; - eta_lims[1] = ts_shift_coord_shifted_log; - - ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); - ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); - - if (fabs(xi_b[1] - xi_b[0]) > tol_xi) - up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, - up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); - - // 2) Add the contribution of the right portion. - xi_b[0] = ts_p2l(inter_pts[0].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - xi_b[1] = ts_p2l(inter_pts[1].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - - eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_lo(up->shift_dir_in_ts_grid)]; - eta_lims[0] = ts_shift_coord_shifted_log; - eta_lims[1] = ts_one; - - ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); - ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); - - if (fabs(xi_b[1] - xi_b[0]) > tol_xi) - up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, - up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); - } - else { - // sNii - // 1) Add the contribution of the left portion. - xi_b[0] = ts_p2l(inter_pts[1].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - xi_b[1] = ts_p2l(inter_pts[0].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - - eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_lo(up->shift_dir_in_ts_grid)]; - eta_lims[0] = ts_shift_coord_shifted_log; - eta_lims[1] = ts_one; - - ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); - ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); - - if (fabs(xi_b[1] - xi_b[0]) > tol_xi) - up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, - up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); - - // 2) Add the contribution of the right portion. - xi_b[0] = ts_p2l(inter_pts[3].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - xi_b[1] = ts_p2l(inter_pts[2].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - - eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_up(up->shift_dir_in_ts_grid)]; - eta_lims[0] = ts_minus_one; - eta_lims[1] = ts_shift_coord_shifted_log; - - ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); - ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); - - if (fabs(xi_b[1] - xi_b[0]) > tol_xi) - up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, - up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); - } -} - -void -ts_subcellint_si_sii(struct gkyl_bc_twistshift *up, struct ts_val_found *inter_pts, - const double *xc_do, const double *xc_tar, const double *cellb_do, const double *cellb_tar, - bool is_upper_shift_dir_cell, const double *shift_c, struct gkyl_mat *mat_do) -{ - // Perform subcell integral si or sii, using fixed x-limits and variable y limits. - // inter_pts: intersections y_{j_tar-/+1/2}-yShift and y_{j_do-/+1/2} (lower/upper y-boundaries of donor cell). - // xc_do: cell center of donor cell. - // xc_tar: cell center of target cell. - // cellb_do: boundaries of target cell. - // cellb_tar: boundaries of target cell. - // is_upper_shift_dir_cell: is the donor the upper cell in the shift dir? - // shift_c: DG coefficients of the shift. - // mat_do: current donor matrix. - - double x_up = cellb_tar[cellb_up(up->shear_dir_in_ts_grid)]; - double x_lo = cellb_tar[cellb_lo(up->shear_dir_in_ts_grid)]; - double shift_lo, shift_up; - - up->shift_func(0.0, (double[]){x_lo}, &shift_lo, up->shift_func_ctx); - up->shift_func(0.0, (double[]){x_up}, &shift_up, up->shift_func_ctx); - - bool is_si = -shift_lo < -shift_up; - - double xi_b[2]; // Limits of xi integral. - if (is_si) { - // si integral. - xi_b[0] = -1.0; - xi_b[1] = ts_p2l(inter_pts[1].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]);; - } - else { - // sii integral. - xi_b[0] = ts_p2l(inter_pts[1].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]);; - xi_b[1] = 1.0; - } - - evalf_t eta_lims[2]; // Table of functions definting the limits of eta integral. - eta_lims[0] = ts_shift_coord_shifted_log; - eta_lims[1] = ts_one; - - struct ts_shift_coord_shifted_log_ctx eta_lims_ctx = { - .shift_coord_tar = cellb_tar[cellb_lo(up->shift_dir_in_ts_grid)], - .shift_sign_fac = 1, - .xc_do = xc_do, - .xc_tar = xc_tar, - .dx = up->ts_grid.dx, - .pick_upper = is_upper_shift_dir_cell, - .shear_dir = up->shear_dir_in_ts_grid, - .shift_dir = up->shift_dir_in_ts_grid, - .shift_dir_bounds = {up->ts_grid.lower[up->shift_dir_in_ts_grid], - up->ts_grid.upper[up->shift_dir_in_ts_grid]}, - .shift_func = up->shift_func, - .shift_func_ctx = up->shift_func_ctx, - }; - - double etalo_xi[up->shift_b.num_basis], etaup_xi[up->shift_b.num_basis]; - ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); - ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); - // Offset between cell centers in direction of the shift. - double xs_off = ts_donor_target_offset(up, xc_do, xc_tar); - - if (fabs(xi_b[1] - xi_b[0]) > tol_xi) - up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, - up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); -} - -void -ts_subcellint_siii_siv(struct gkyl_bc_twistshift *up, struct ts_val_found *inter_pts, - const double *xc_do, const double *xc_tar, const double *cellb_do, const double *cellb_tar, - bool is_upper_shift_dir_cell, const double *shift_c, struct gkyl_mat *mat_do) -{ - // Perform subcell integral siii or siv, using fixed x-limits and variable y limits. - // inter_pts: intersections y_{j_tar-/+1/2}-yShift and y_{j_do-/+1/2} (lower/upper y-boundaries of donor cell). - // xc_do: cell center of donor cell. - // xc_tar: cell center of target cell. - // cellb_do: boundaries of target cell. - // cellb_tar: boundaries of target cell. - // is_upper_shift_dir_cell: is the donor the upper cell in the shift dir? - // shift_c: DG coefficients of the shift. - // mat_do: current donor matrix. - - double x_lo = cellb_tar[cellb_lo(up->shear_dir_in_ts_grid)]; - double x_up = cellb_tar[cellb_up(up->shear_dir_in_ts_grid)]; - double shift_lo, shift_up; - - up->shift_func(0.0, (double[]){x_lo}, &shift_lo, up->shift_func_ctx); - up->shift_func(0.0, (double[]){x_up}, &shift_up, up->shift_func_ctx); - - bool is_siii = -shift_lo > -shift_up; - - double xi_b[2]; // Limits of xi integral. - if (is_siii) { - // siii integral. - xi_b[0] = -1.0; - xi_b[1] = ts_p2l(inter_pts[2].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]);; - } - else { - // siv integral. - xi_b[0] = ts_p2l(inter_pts[2].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]);; - xi_b[1] = 1.0; - } - - evalf_t eta_lims[2]; // Table of functions definting the limits of eta integral. - eta_lims[0] = ts_minus_one; - eta_lims[1] = ts_shift_coord_shifted_log; - - struct ts_shift_coord_shifted_log_ctx eta_lims_ctx = { - .shift_coord_tar = cellb_tar[cellb_up(up->shift_dir_in_ts_grid)], - .shift_sign_fac = 1, - .xc_do = xc_do, - .xc_tar = xc_tar, - .dx = up->ts_grid.dx, - .pick_upper = is_upper_shift_dir_cell, - .shear_dir = up->shear_dir_in_ts_grid, - .shift_dir = up->shift_dir_in_ts_grid, - .shift_dir_bounds = {up->ts_grid.lower[up->shift_dir_in_ts_grid], - up->ts_grid.upper[up->shift_dir_in_ts_grid]}, - .shift_func = up->shift_func, - .shift_func_ctx = up->shift_func_ctx, - }; - - double etalo_xi[up->shift_b.num_basis], etaup_xi[up->shift_b.num_basis]; - ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); - ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); - // Offset between cell centers in direction of the shift. - double xs_off = ts_donor_target_offset(up, xc_do, xc_tar); - - if (fabs(xi_b[1] - xi_b[0]) > tol_xi) - up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, - up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); -} - -void -ts_subcellint_sv_svi(struct gkyl_bc_twistshift *up, struct ts_val_found *inter_pts, - const double *xc_do, const double *xc_tar, const double *cellb_do, const double *cellb_tar, - bool is_upper_shift_dir_cell, const double *shift_c, struct gkyl_mat *mat_do) -{ - // Perform sv or svi subcell integrals. - // inter_pts: intersections y_{j_tar-/+1/2}-yShift and y_{j_do-/+1/2} (lower/upper y-boundaries of donor cell). - // xc_do: cell center of donor cell. - // xc_tar: cell center of target cell. - // cellb_do: boundaries of target cell. - // cellb_tar: boundaries of target cell. - // is_upper_shift_dir_cell: is the donor the upper cell in the shift dir? - // shift_c: DG coefficients of the shift. - // mat_do: current donor matrix. - - bool is_sv = inter_pts[3].value < inter_pts[1].value; - - struct ts_shift_coord_shifted_log_ctx eta_lims_ctx = { - .shift_sign_fac = 1, - .xc_do = xc_do, - .xc_tar = xc_tar, - .dx = up->ts_grid.dx, - .pick_upper = is_upper_shift_dir_cell, - .shear_dir = up->shear_dir_in_ts_grid, - .shift_dir = up->shift_dir_in_ts_grid, - .shift_dir_bounds = {up->ts_grid.lower[up->shift_dir_in_ts_grid], - up->ts_grid.upper[up->shift_dir_in_ts_grid]}, - .shift_func = up->shift_func, - .shift_func_ctx = up->shift_func_ctx, - }; - - double xi_b[2]; // Limits of xi integral. - evalf_t eta_lims[2]; // Table of functions definting the limits of eta integral. - double etalo_xi[up->shift_b.num_basis], etaup_xi[up->shift_b.num_basis]; - - // Offset between cell centers in direction of the shift. - double xs_off = ts_donor_target_offset(up, xc_do, xc_tar); - - if (is_sv) { - // sv - // 1) Add the contribution of the left portion. - xi_b[0] = -1.0; - xi_b[1] = ts_p2l(inter_pts[3].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - - eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_up(up->shift_dir_in_ts_grid)]; - eta_lims[0] = ts_minus_one; - eta_lims[1] = ts_shift_coord_shifted_log; - - ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); - ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); - - if (fabs(xi_b[1] - xi_b[0]) > tol_xi) - up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, - up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); - - // 2) Add the contribution of the right portion. - xi_b[0] = ts_p2l(inter_pts[0].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - xi_b[1] = ts_p2l(inter_pts[1].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - - eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_lo(up->shift_dir_in_ts_grid)]; - eta_lims[0] = ts_shift_coord_shifted_log; - eta_lims[1] = ts_one; - - ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); - ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); - - if (fabs(xi_b[1] - xi_b[0]) > tol_xi) - up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, - up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); - } - else { - // svi - // 1) Add the contribution of the left portion. - xi_b[0] = ts_p2l(inter_pts[1].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - xi_b[1] = ts_p2l(inter_pts[0].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - - eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_lo(up->shift_dir_in_ts_grid)]; - eta_lims[0] = ts_shift_coord_shifted_log; - eta_lims[1] = ts_one; - - ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); - ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); - - if (fabs(xi_b[1] - xi_b[0]) > tol_xi) - up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, - up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); - - // 2) Add the contribution of the right portion. - xi_b[0] = ts_p2l(inter_pts[3].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - xi_b[1] = 1.0; - - eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_up(up->shift_dir_in_ts_grid)]; - eta_lims[0] = ts_minus_one; - eta_lims[1] = ts_shift_coord_shifted_log; - - ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); - ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); - - if (fabs(xi_b[1] - xi_b[0]) > tol_xi) - up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, - up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); - } -} - -void -ts_subcellint_svii_sviii(struct gkyl_bc_twistshift *up, struct ts_val_found *inter_pts, - const double *xc_do, const double *xc_tar, const double *cellb_do, const double *cellb_tar, - bool is_upper_shift_dir_cell, const double *shift_c, struct gkyl_mat *mat_do) -{ - // Perform svii or sviii subcell integrals. - // inter_pts: intersections y_{j_tar-/+1/2}-yShift and y_{j_do-/+1/2} (lower/upper y-boundaries of donor cell). - // xc_do: cell center of donor cell. - // xc_tar: cell center of target cell. - // cellb_do: boundaries of target cell. - // cellb_tar: boundaries of target cell. - // is_upper_shift_dir_cell: is the donor the upper cell in the shift dir? - // shift_c: DG coefficients of the shift. - // mat_do: current donor matrix. - - bool is_svii = inter_pts[0].value < inter_pts[2].value; - - struct ts_shift_coord_shifted_log_ctx eta_lims_ctx = { - .shift_sign_fac = 1, - .xc_do = xc_do, - .xc_tar = xc_tar, - .dx = up->ts_grid.dx, - .pick_upper = is_upper_shift_dir_cell, - .shear_dir = up->shear_dir_in_ts_grid, - .shift_dir = up->shift_dir_in_ts_grid, - .shift_dir_bounds = {up->ts_grid.lower[up->shift_dir_in_ts_grid], - up->ts_grid.upper[up->shift_dir_in_ts_grid]}, - .shift_func = up->shift_func, - .shift_func_ctx = up->shift_func_ctx, - }; - - double xi_b[2]; // Limits of xi integral. - evalf_t eta_lims[2]; // Table of functions definting the limits of eta integral. - double etalo_xi[up->shift_b.num_basis], etaup_xi[up->shift_b.num_basis]; - - // Offset between cell centers in direction of the shift. - double xs_off = ts_donor_target_offset(up, xc_do, xc_tar); - - if (is_svii) { - // svii - // 1) Add the contribution of the left portion. - xi_b[0] = -1.0; - xi_b[1] = ts_p2l(inter_pts[0].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - - eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_lo(up->shift_dir_in_ts_grid)]; - eta_lims[0] = ts_shift_coord_shifted_log; - eta_lims[1] = ts_one; - - ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); - ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); - - if (fabs(xi_b[1] - xi_b[0]) > tol_xi) - up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, - up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); - - // 2) Add the contribution of the right portion. - xi_b[0] = ts_p2l(inter_pts[3].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - xi_b[1] = ts_p2l(inter_pts[2].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - - eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_up(up->shift_dir_in_ts_grid)]; - eta_lims[0] = ts_minus_one; - eta_lims[1] = ts_shift_coord_shifted_log; - - ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); - ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); - - if (fabs(xi_b[1] - xi_b[0]) > tol_xi) - up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, - up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); - } - else { - // sviii - // 1) Add the contribution of the left portion. - xi_b[0] = ts_p2l(inter_pts[2].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - xi_b[1] = ts_p2l(inter_pts[3].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - - eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_up(up->shift_dir_in_ts_grid)]; - eta_lims[0] = ts_minus_one; - eta_lims[1] = ts_shift_coord_shifted_log; - - ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); - ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); - - if (fabs(xi_b[1] - xi_b[0]) > tol_xi) - up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, - up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); - - // 2) Add the contribution of the right portion. - xi_b[0] = ts_p2l(inter_pts[0].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - xi_b[1] = 1.0; - - eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_lo(up->shift_dir_in_ts_grid)]; - eta_lims[0] = ts_shift_coord_shifted_log; - eta_lims[1] = ts_one; - - ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); - ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); - - if (fabs(xi_b[1] - xi_b[0]) > tol_xi) - up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, - up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); - } -} - -void -ts_subcellint_six_sx(struct gkyl_bc_twistshift *up, struct ts_val_found *inter_pts, - const double *xc_do, const double *xc_tar, const double *cellb_do, const double *cellb_tar, - bool is_upper_shift_dir_cell, const double *shift_c, struct gkyl_mat *mat_do) -{ - // Perform six or sx subcell integrals. - // inter_pts: intersections y_{j_tar-/+1/2}-yShift and y_{j_do-/+1/2} (lower/upper y-boundaries of donor cell). - // xc_do: cell center of donor cell. - // xc_tar: cell center of target cell. - // cellb_do: boundaries of target cell. - // cellb_tar: boundaries of target cell. - // is_upper_shift_dir_cell: is the donor the upper cell in the shift dir? - // shift_c: DG coefficients of the shift. - // mat_do: current donor matrix. - - bool is_six = inter_pts[0].value < inter_pts[1].value; - - // Limits of xi integral. - double xi_b[2]; - if (is_six) { - // six - xi_b[0] = ts_p2l(inter_pts[0].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - xi_b[1] = ts_p2l(inter_pts[1].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - } - else { - // sx - xi_b[0] = ts_p2l(inter_pts[1].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - xi_b[1] = ts_p2l(inter_pts[0].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - } - - struct ts_shift_coord_shifted_log_ctx eta_lims_ctx = { - .shift_coord_tar = cellb_tar[cellb_lo(up->shift_dir_in_ts_grid)], - .shift_sign_fac = 1, - .xc_do = xc_do, - .xc_tar = xc_tar, - .dx = up->ts_grid.dx, - .pick_upper = is_upper_shift_dir_cell, - .shear_dir = up->shear_dir_in_ts_grid, - .shift_dir = up->shift_dir_in_ts_grid, - .shift_dir_bounds = {up->ts_grid.lower[up->shift_dir_in_ts_grid], - up->ts_grid.upper[up->shift_dir_in_ts_grid]}, - .shift_func = up->shift_func, - .shift_func_ctx = up->shift_func_ctx, - }; - - evalf_t eta_lims[2]; // Table of functions definting the limits of eta integral. - eta_lims[0] = ts_shift_coord_shifted_log; - eta_lims[1] = ts_one; - - double etalo_xi[up->shift_b.num_basis], etaup_xi[up->shift_b.num_basis]; - ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); - ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); - // Offset between cell centers in direction of the shift. - double xs_off = ts_donor_target_offset(up, xc_do, xc_tar); - - if (fabs(xi_b[1] - xi_b[0]) > tol_xi) - up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, - up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); -} - -void -ts_subcellint_sxi_sxii(struct gkyl_bc_twistshift *up, struct ts_val_found *inter_pts, - const double *xc_do, const double *xc_tar, const double *cellb_do, const double *cellb_tar, - bool is_upper_shift_dir_cell, const double *shift_c, struct gkyl_mat *mat_do) -{ - // Perform sxi or sxii subcell integrals. - // inter_pts: intersections y_{j_tar-/+1/2}-yShift and y_{j_do-/+1/2} (lower/upper y-boundaries of donor cell). - // xc_do: cell center of donor cell. - // xc_tar: cell center of target cell. - // cellb_do: boundaries of target cell. - // cellb_tar: boundaries of target cell. - // is_upper_shift_dir_cell: is the donor the upper cell in the shift dir? - // shift_c: DG coefficients of the shift. - // mat_do: current donor matrix. - - bool is_sxi = inter_pts[3].value < inter_pts[2].value; - - // Limits of xi integral. - double xi_b[2]; - if (is_sxi) { - // six - xi_b[0] = ts_p2l(inter_pts[3].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - xi_b[1] = ts_p2l(inter_pts[2].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - } - else { - // sx - xi_b[0] = ts_p2l(inter_pts[2].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - xi_b[1] = ts_p2l(inter_pts[3].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - } - - struct ts_shift_coord_shifted_log_ctx eta_lims_ctx = { - .shift_coord_tar = cellb_tar[cellb_up(up->shift_dir_in_ts_grid)], - .shift_sign_fac = 1, - .xc_do = xc_do, - .xc_tar = xc_tar, - .dx = up->ts_grid.dx, - .pick_upper = is_upper_shift_dir_cell, - .shear_dir = up->shear_dir_in_ts_grid, - .shift_dir = up->shift_dir_in_ts_grid, - .shift_dir_bounds = {up->ts_grid.lower[up->shift_dir_in_ts_grid], - up->ts_grid.upper[up->shift_dir_in_ts_grid]}, - .shift_func = up->shift_func, - .shift_func_ctx = up->shift_func_ctx, - }; - - evalf_t eta_lims[2]; // Table of functions definting the limits of eta integral. - eta_lims[0] = ts_minus_one; - eta_lims[1] = ts_shift_coord_shifted_log; - - double etalo_xi[up->shift_b.num_basis], etaup_xi[up->shift_b.num_basis]; - ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); - ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); - // Offset between cell centers in direction of the shift. - double xs_off = ts_donor_target_offset(up, xc_do, xc_tar); - - if (fabs(xi_b[1] - xi_b[0]) > tol_xi) - up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, - up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); -} - -void -ts_subcellint_sxiii_sxiv(struct gkyl_bc_twistshift *up, struct ts_val_found *inter_pts, - const double *xc_do, const double *xc_tar, const double *cellb_do, const double *cellb_tar, - bool is_upper_shift_dir_cell, const double *shift_c, struct gkyl_mat *mat_do) -{ - // Perform sxiii or sxiv subcell integrals. - // inter_pts: intersections y_{j_tar-/+1/2}-yShift and y_{j_do-/+1/2} (lower/upper y-boundaries of donor cell). - // xc_do: cell center of donor cell. - // xc_tar: cell center of target cell. - // cellb_do: boundaries of target cell. - // cellb_tar: boundaries of target cell. - // is_upper_shift_dir_cell: is the donor the upper cell in the shift dir? - // shift_c: DG coefficients of the shift. - // mat_do: current donor matrix. - - double x_lo = cellb_tar[cellb_lo(up->shear_dir_in_ts_grid)]; - double x_up = cellb_tar[cellb_up(up->shear_dir_in_ts_grid)]; - double shift_lo, shift_up; - - up->shift_func(0.0, (double[]){x_lo}, &shift_lo, up->shift_func_ctx); - up->shift_func(0.0, (double[]){x_up}, &shift_up, up->shift_func_ctx); - - bool is_sxiii = -shift_lo < -shift_up; - - struct ts_shift_coord_shifted_log_ctx eta_lims_ctx = { - .shift_sign_fac = 1, - .xc_do = xc_do, - .xc_tar = xc_tar, - .dx = up->ts_grid.dx, - .pick_upper = is_upper_shift_dir_cell, - .shear_dir = up->shear_dir_in_ts_grid, - .shift_dir = up->shift_dir_in_ts_grid, - .shift_dir_bounds = {up->ts_grid.lower[up->shift_dir_in_ts_grid], - up->ts_grid.upper[up->shift_dir_in_ts_grid]}, - .shift_func = up->shift_func, - .shift_func_ctx = up->shift_func_ctx, - }; - - double xi_b[2]; // Limits of xi integral. - evalf_t eta_lims[2]; // Table of functions definting the limits of eta integral. - double etalo_xi[up->shift_b.num_basis], etaup_xi[up->shift_b.num_basis]; - - // Offset between cell centers in direction of the shift. - double xs_off = ts_donor_target_offset(up, xc_do, xc_tar); - - // 1) Add the contribution of the left portion. - xi_b[0] = -1.0; - xi_b[1] = ts_p2l(inter_pts[3].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - - if (is_sxiii) { - eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_up(up->shift_dir_in_ts_grid)]; - eta_lims[0] = ts_minus_one; - eta_lims[1] = ts_shift_coord_shifted_log; - } - else { - eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_lo(up->shift_dir_in_ts_grid)]; - eta_lims[0] = ts_shift_coord_shifted_log; - eta_lims[1] = ts_one; - } - - ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); - ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); - - if (fabs(xi_b[1] - xi_b[0]) > tol_xi) - up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, - up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); - - // 2) Add the contribution of the right portion. - xi_b[0] = ts_p2l(inter_pts[0].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); - xi_b[1] = 1.0; - - if (is_sxiii) { - eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_lo(up->shift_dir_in_ts_grid)]; - eta_lims[0] = ts_shift_coord_shifted_log; - eta_lims[1] = ts_one; - } - else { - eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_up(up->shift_dir_in_ts_grid)]; - eta_lims[0] = ts_minus_one; - eta_lims[1] = ts_shift_coord_shifted_log; - } - - ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); - ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); - - if (fabs(xi_b[1] - xi_b[0]) > tol_xi) - up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, - up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); -} - -void -ts_subcellint_sxv_sxvi(struct gkyl_bc_twistshift *up, struct ts_val_found *inter_pts, - const double *xc_do, const double *xc_tar, const double *cellb_do, const double *cellb_tar, - bool is_upper_shift_dir_cell, const double *shift_c, struct gkyl_mat *mat_do) -{ - // Perform subcell integral sxv or sxvi, using fixed x-limits and variable y limits. - // inter_pts: intersections y_{j_tar-/+1/2}-yShift and y_{j_do-/+1/2} (lower/upper y-boundaries of donor cell). - // xc_do: cell center of donor cell. - // xc_tar: cell center of target cell. - // cellb_do: boundaries of target cell. - // cellb_tar: boundaries of target cell. - // is_upper_shift_dir_cell: is the donor the upper cell in the shift dir? - // shift_c: DG coefficients of the shift. - // mat_do: current donor matrix. - - double xi_b[] = {-1.0, 1.0}; // Limits of xi integral. - - double shift_dir_bounds[] = {up->ts_grid.lower[up->shift_dir_in_ts_grid], - up->ts_grid.upper[up->shift_dir_in_ts_grid]}; - - double x_eval = xc_do[up->shear_dir_in_ts_grid]; - double shift; - - up->shift_func(0.0, (double[]){x_eval}, &shift, up->shift_func_ctx); - - double shifted_coord = cellb_tar[cellb_lo(up->shift_dir_in_ts_grid)] - shift; - shifted_coord = ts_wrap_to_range(shifted_coord, shift_dir_bounds[0], shift_dir_bounds[1], - is_upper_shift_dir_cell); - - struct ts_shift_coord_shifted_log_ctx eta_lims_ctx = { - .shift_sign_fac = 1, - .xc_do = xc_do, - .xc_tar = xc_tar, - .dx = up->ts_grid.dx, - .pick_upper = is_upper_shift_dir_cell, - .shear_dir = up->shear_dir_in_ts_grid, - .shift_dir = up->shift_dir_in_ts_grid, - .shift_dir_bounds = {up->ts_grid.lower[up->shift_dir_in_ts_grid], - up->ts_grid.upper[up->shift_dir_in_ts_grid]}, - .shift_func = up->shift_func, - .shift_func_ctx = up->shift_func_ctx, - }; - - evalf_t eta_lims[2]; // Table of functions definting the limits of eta integral. - if ( cellb_do[cellb_lo(up->shift_dir_in_ts_grid)] <= shifted_coord && - shifted_coord <= cellb_do[cellb_up(up->shift_dir_in_ts_grid)] ) { - // sxv integral. - eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_lo(up->shift_dir_in_ts_grid)]; - eta_lims[0] = ts_shift_coord_shifted_log; - eta_lims[1] = ts_one; - } - else { - // sxvi integral. - eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_up(up->shift_dir_in_ts_grid)]; - eta_lims[0] = ts_minus_one; - eta_lims[1] = ts_shift_coord_shifted_log; - } - - double etalo_xi[up->shift_b.num_basis], etaup_xi[up->shift_b.num_basis]; - ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); - ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); - // Offset between cell centers in direction of the shift. - double xs_off = ts_donor_target_offset(up, xc_do, xc_tar); - - if (fabs(xi_b[1] - xi_b[0]) > tol_xi) - up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, - up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); -} - -struct gkyl_nmat * -ts_calc_mats(struct gkyl_bc_twistshift *up) -{ - - // Allocate matrices containing the discrete subcell integrals. - int num_do_tot = 0; - for (int i=0; ishear_r.volume; i++) - num_do_tot += up->num_do[i]; - - struct gkyl_nmat *matsdo = gkyl_nmat_new(num_do_tot, up->basis.num_basis, up->basis.num_basis); - for (int n=0; nnum; ++n) { - struct gkyl_mat mat = gkyl_nmat_get(matsdo, n); - for (int j=0; jnc; ++j) - for (int i=0; inr; ++i) - gkyl_mat_set(&mat, i, j, 0.0); - } - - // y-index of the reference target used to precalc matrices. For positive(negative) - // yShift idx=1(last) might be better, but ideally it shouldn't matter. - int shift_dir_idx_tar = 1; - - double shift_dir_lims[] = {up->ts_grid.lower[up->shift_dir_in_ts_grid], - up->ts_grid.upper[up->shift_dir_in_ts_grid]}; - - // Create an eval_on_nodes updater to use its nodes and functions (but not - // the whole advance method). - up->ev_on_nod1d = gkyl_eval_on_nodes_new(&up->shear_grid, &up->shift_b, 1, ts_one, NULL); - // Create an array to store evaluations of a function at 1D nodes. - up->func_nod1d = gkyl_array_new(GKYL_DOUBLE, 1, up->shift_b.num_basis); - - struct gkyl_range_iter iter; - gkyl_range_iter_init(&iter, &up->shear_r); - while (gkyl_range_iter_next(&iter)) { - - // Get the cell boundaries and cell center. - int idx_tar[2]; // Target index. - idx_tar[up->shift_dir_in_ts_grid] = shift_dir_idx_tar; - idx_tar[up->shear_dir_in_ts_grid] = iter.idx[0]; - double cellb_tar[4] = {0.0}; // Cell boundaries, x lo and up, y lo and up; - double xc_tar[2] = {0.0}; // Cell center. - ts_grid_cell_boundaries(&up->ts_grid, idx_tar, cellb_tar); - gkyl_rect_grid_cell_center(&up->ts_grid, idx_tar, xc_tar); - - long shift_loc = gkyl_range_idx(&up->shear_r, iter.idx); - double *shift_c = (double *) gkyl_array_fetch(up->shift_dg, shift_loc); - - long linidx_do = ts_shift_dir_idx_do_linidx(up->num_do, iter.idx[0], shift_dir_idx_tar, - up->ts_grid.cells[up->shift_dir_in_ts_grid], up->shear_r.lower[0]); - int *shift_dir_idx_do_ptr = &up->shift_dir_idx_do[linidx_do]; - - long linidx_mats_do = 0; - for (int i=0; ishear_r.lower[0]; i++) - linidx_mats_do += up->num_do[i]; - - // Check that the shift variation within this x-cell < Ly. - // The algorithm assumes at most one x-intersection per (y-boundary, y-boundary) pair, - // which breaks when |S(x_up) - S(x_lo)| >= Ly. - double x_lo = cellb_tar[cellb_lo(up->shear_dir_in_ts_grid)]; - double x_up = cellb_tar[cellb_up(up->shear_dir_in_ts_grid)]; - double Ly = shift_dir_lims[1] - shift_dir_lims[0]; - double S_lo, S_up, S_c; - up->shift_func(0.0, (double[]){x_lo}, &S_lo, up->shift_func_ctx); - up->shift_func(0.0, (double[]){x_up}, &S_up, up->shift_func_ctx); - up->shift_func(0.0, (double[]){xc_tar[up->shear_dir_in_ts_grid]}, &S_c, up->shift_func_ctx); - - if (fabs(S_up - S_lo) >= Ly) { - fprintf(stderr, "bc_twistshift: shift variation |S(x_up)-S(x_lo)| = %g across a single x-cell" - " exceeds Ly = %g (cell ix=%d). Increase Nx, reduce the shear, or increase Ly.\n", - fabs(S_up - S_lo), Ly, iter.idx[0]); - assert(false); - } - - for (int iC=0; iCnum_do[iter.idx[0]-up->shear_r.lower[0]]; iC++){ - int idx_do[2]; // Target index. - idx_do[up->shift_dir_in_ts_grid] = shift_dir_idx_do_ptr[iC]; - idx_do[up->shear_dir_in_ts_grid] = iter.idx[0]; - - double cellb_do[4] = {0.0}; // Cell boundaries, x lo and up, y lo and up; - double xc_do[2] = {0.0}; // Cell center. - ts_grid_cell_boundaries(&up->ts_grid, idx_do, cellb_do); - gkyl_rect_grid_cell_center(&up->ts_grid, idx_do, xc_do); - - // Get the matrix we are presently assigning. - struct gkyl_mat mat_do = gkyl_nmat_get(matsdo, linidx_mats_do+iC); - - // Periodic copy in which to find the target for this donor-target pair: the integer nP such that - // S_c \approx y_tar_c - y_do_c + nP*Ly. All 4 inter_pts must use this same nP so that only - // roots from the physical intersection are accepted (not roots from other periodic copies). - int nP_primary = (int)round((S_c - (xc_tar[up->shift_dir_in_ts_grid] - xc_do[up->shift_dir_in_ts_grid])) / Ly); - - // Find the points where y_{j_tar-/+1/2}-yShift intersect the y=y_{j_do-/+1/2} lines. - // Also record the number and indices of points found/not found. - struct ts_val_found inter_pts[4] = {}; - int num_inter_pts_found = 0, num_inter_pts_not_found = 4; - int inter_pts_found_idxs[4], inter_pts_not_found_idxs[4]; - for (int i=0; i<2; i++) { // Loop over j_tar-/+1/2 - for (int j=0; j<2; j++) { // Loop over j_do-/+1/2 - double shift_dir_coord_tar = cellb_tar[2*up->shift_dir_in_ts_grid+i]; - double shift_dir_coord_do = cellb_do[2*up->shift_dir_in_ts_grid+j]; - struct gkyl_qr_res inter_res = ts_find_intersect(up, shift_dir_coord_tar, shift_dir_coord_do, - (double[]) {cellb_tar[cellb_lo(up->shear_dir_in_ts_grid)],cellb_tar[cellb_up(up->shear_dir_in_ts_grid)]}, - shift_dir_lims, nP_primary); - int ip_linc = i*2+j; - inter_pts[ip_linc].status = inter_res.status == 0; - if (inter_res.status == 0) { - inter_pts[ip_linc].value = inter_res.res; - inter_pts_found_idxs[num_inter_pts_found] = ip_linc; - num_inter_pts_found++; - } - else { - inter_pts_not_found_idxs[num_inter_pts_not_found] = ip_linc; - num_inter_pts_not_found--; - } - } - } - - bool is_upper_shift_dir_cell = idx_do[up->shift_dir_in_ts_grid] == up->ts_grid.cells[up->shift_dir_in_ts_grid]; - - if (num_inter_pts_found == 4) { - // sN: all intersections are found at this cell. - ts_subcellint_sNi_sNii(up, inter_pts, xc_do, xc_tar, cellb_do, cellb_tar, is_upper_shift_dir_cell, shift_c, &mat_do); - } - else if (num_inter_pts_found == 1) { - if (inter_pts[1].status) { - // si: y_{j_tar-1/2}-yShift intersects x_{i-1/2}. - // sii: y_{j_tar-1/2}-yShift intersects x_{i+1/2}. - ts_subcellint_si_sii(up, inter_pts, xc_do, xc_tar, cellb_do, cellb_tar, is_upper_shift_dir_cell, shift_c, &mat_do); - } - else { - // siii: y_{j_tar+1/2}-yShift intersects x_{i-1/2}. - // siv: y_{j_tar+1/2}-yShift intersects x_{i+1/2}. - ts_subcellint_siii_siv(up, inter_pts, xc_do, xc_tar, cellb_do, cellb_tar, is_upper_shift_dir_cell, shift_c, &mat_do); - } - } - else if (num_inter_pts_found == 3) { - if (!inter_pts[2].status) { - // sv: y_{j_tar+1/2}-yShift doesn't intersect y_{j_do-1/2} & intersects x_{i-1/2}. - // svi: y_{j_tar+1/2}-yShift doesn't intersect y_{j_do-1/2} & intersects x_{i+1/2}. - ts_subcellint_sv_svi(up, inter_pts, xc_do, xc_tar, cellb_do, cellb_tar, is_upper_shift_dir_cell, shift_c, &mat_do); - } - else { - // svii: y_{j_tar-1/2}-yShift doesn't intersect y_{j_do+1/2} & intersects x_{i-1/2}. - // sviii: y_{j_tar-1/2}-yShift doesn't intersect y_{j_do+1/2} & intersects x_{i+1/2}. - ts_subcellint_svii_sviii(up, inter_pts, xc_do, xc_tar, cellb_do, cellb_tar, is_upper_shift_dir_cell, shift_c, &mat_do); - } - } - else if (num_inter_pts_found == 2) { - if (inter_pts[0].status && inter_pts[1].status) { - // six: y_{j_tar-1/2}-yShift crosses y_{j_do-/+1/2} (increasing yShift). - // sx: y_{j_tar-1/2}-yShift crosses y_{j_do-/+1/2} (decreasing yShift). - ts_subcellint_six_sx(up, inter_pts, xc_do, xc_tar, cellb_do, cellb_tar, is_upper_shift_dir_cell, shift_c, &mat_do); - } - else if (inter_pts[2].status && inter_pts[3].status) { - // sxi: y_{j_tar+1/2}-yShift crosses y_{j_do-/+1/2} (decreasing yShift). - // sxii: y_{j_tar+1/2}-yShift crosses y_{j_do-/+1/2} (increasing yShift). - ts_subcellint_sxi_sxii(up, inter_pts, xc_do, xc_tar, cellb_do, cellb_tar, is_upper_shift_dir_cell, shift_c, &mat_do); - } - else { - // sxiii: y_{j_tar-1/2}-yShift crosses y_{j_do-1/2} & y_{j_tar+1/2}-yShift crosses y_{j_do+1/2} (increasing yShift). - // sxiv: y_{j_tar-1/2}-yShift crosses y_{j_do-1/2} & y_{j_tar+1/2}-yShift crosses y_{j_do+1/2} (decreasing yShift). - ts_subcellint_sxiii_sxiv(up, inter_pts, xc_do, xc_tar, cellb_do, cellb_tar, is_upper_shift_dir_cell, shift_c, &mat_do); - } - } - else if (num_inter_pts_found == 0) { - // sxv: y_{j_tar-1/2}-yShift crosses x_{i-/+1/2}. - // sxvi: y_{j_tar+1/2}-yShift crosses x_{i-/+1/2}. - ts_subcellint_sxv_sxvi(up, inter_pts, xc_do, xc_tar, cellb_do, cellb_tar, is_upper_shift_dir_cell, shift_c, &mat_do); - } - else { - // An error occurred. This shouldn't happen. - assert(false); - } - } - - } - - gkyl_array_release(up->func_nod1d); - gkyl_eval_on_nodes_release(up->ev_on_nod1d); - - struct gkyl_nmat *matsdo_out = up->use_gpu? gkyl_nmat_cu_dev_new(matsdo->num, matsdo->nr, matsdo->nc) - : gkyl_nmat_acquire(matsdo); - gkyl_nmat_copy(matsdo_out, matsdo); - gkyl_nmat_release(matsdo); - - return matsdo_out; -} - -long * -ts_calc_num_numcol_fidx_do(struct gkyl_bc_twistshift *up) -{ - // Calculate the linear indices into the donor distribution function gkyl_array - // for each num-numcol plane (in the num-numcol-num_basis) space. - - long *num_numcol_fidx_do_ho = (long*) gkyl_malloc(up->fmat->num * up->fmat->nc * sizeof(long)); - - // Location in the direction of the BC from which to take donor - // distributions. We assume that the user filled the ghost cell with the skin - // on the other side (i.e. applied periodicity first). - int bc_dir_loc_do = up->edge == GKYL_LOWER_EDGE? up->local_bcdir_ext_r.lower[up->bc_dir] - : up->local_bcdir_ext_r.upper[up->bc_dir]; - - // Range over directions other than shear and bc dirs. - struct gkyl_range shearbc_perp_r; - int remove[GKYL_MAX_DIM] = {0}, loc_in_dir[GKYL_MAX_DIM] = {0};; - remove[up->shear_dir] = remove[up->bc_dir] = 1; - loc_in_dir[up->shear_dir] = up->local_bcdir_ext_r.lower[up->shear_dir]; - loc_in_dir[up->bc_dir] = bc_dir_loc_do; - gkyl_range_deflate(&shearbc_perp_r, &up->local_bcdir_ext_r, remove, loc_in_dir); - - int shift_dir_in_shearbc_perp_r; - if (up->shift_dir < up->shear_dir && up->shift_dir < up->bc_dir) - shift_dir_in_shearbc_perp_r = up->shift_dir; - else if (up->shift_dir > up->shear_dir && up->shift_dir > up->bc_dir) - shift_dir_in_shearbc_perp_r = up->shift_dir-2; - else - shift_dir_in_shearbc_perp_r = up->shift_dir-1; - - int prev_shift_dir_idx = 0; - int donor_count = 0; - int do_idx[up->local_bcdir_ext_r.ndim]; - - // Loop over directions perpendicular to shear and BC dirs. - struct gkyl_range_iter iter; - gkyl_range_iter_init(&iter, &shearbc_perp_r); - while (gkyl_range_iter_next(&iter)) { - int ic = 0; - for (int d=0; dlocal_bcdir_ext_r.ndim; d++) { - if (d != up->bc_dir && d != up->shear_dir) { - do_idx[d] = iter.idx[ic]; - ic++; - } - } - do_idx[up->bc_dir] = bc_dir_loc_do; - - struct gkyl_range_iter shear_dir_iter; - gkyl_range_iter_init(&shear_dir_iter, &up->shear_r); - while (gkyl_range_iter_next(&shear_dir_iter)) { - - int shear_dir_idx = shear_dir_iter.idx[0]; - - long linidx_do = ts_shift_dir_idx_do_linidx(up->num_do, shear_dir_idx, - iter.idx[shift_dir_in_shearbc_perp_r], up->ts_grid.cells[up->shift_dir_in_ts_grid], up->shear_r.lower[0]); - - for (int i = 0; i < up->num_do[shear_dir_idx-up->shear_r.lower[0]]; i++) { - do_idx[up->shear_dir] = shear_dir_idx; - do_idx[up->shift_dir] = up->shift_dir_idx_do[linidx_do+i]; - - long loc = gkyl_range_idx(&up->local_bcdir_ext_r, do_idx); - num_numcol_fidx_do_ho[donor_count] = loc; - - donor_count += 1; - } - } - prev_shift_dir_idx = iter.idx[shift_dir_in_shearbc_perp_r]; - } - - long *num_numcol_fidx_do; - if (!up->use_gpu) { - num_numcol_fidx_do = (long*) gkyl_malloc(up->fmat->num * up->fmat->nc * sizeof(long)); - memcpy(num_numcol_fidx_do, num_numcol_fidx_do_ho, up->fmat->num * up->fmat->nc * sizeof(long)); - } -#ifdef GKYL_HAVE_CUDA - if (up->use_gpu) { - num_numcol_fidx_do = (long*) gkyl_cu_malloc(up->fmat->num * up->fmat->nc * sizeof(long)); - gkyl_cu_memcpy(num_numcol_fidx_do, num_numcol_fidx_do_ho, up->fmat->num * up->fmat->nc * sizeof(long), GKYL_CU_MEMCPY_H2D); - } -#endif - - gkyl_free(num_numcol_fidx_do_ho); - - return num_numcol_fidx_do; -} - -long * -ts_calc_num_numcol_fidx_tar(struct gkyl_bc_twistshift *up) -{ - - long *num_numcol_fidx_tar_ho = (long*) gkyl_malloc(up->fmat->num * up->fmat->nc * sizeof(long)); - - // Location in the direction of the BC in which to place the target - // distributions. We assume that the local_bcdir_ext_r is a range extended in z (it - // includes the z ghose cell). - int bc_dir_loc_tar = up->edge == GKYL_LOWER_EDGE? up->local_bcdir_ext_r.lower[up->bc_dir] - : up->local_bcdir_ext_r.upper[up->bc_dir]; - - // Range over directions other than shear and bc dirs. - struct gkyl_range shearbc_perp_r; - int remove[GKYL_MAX_DIM] = {0}, loc_in_dir[GKYL_MAX_DIM] = {0};; - remove[up->shear_dir] = remove[up->bc_dir] = 1; - loc_in_dir[up->shear_dir] = up->local_bcdir_ext_r.lower[up->shear_dir]; - loc_in_dir[up->bc_dir] = bc_dir_loc_tar; - gkyl_range_deflate(&shearbc_perp_r, &up->local_bcdir_ext_r, remove, loc_in_dir); - - int tar_idx[up->local_bcdir_ext_r.ndim]; - int tar_count = 0; - - // Loop over directions perpendicular to shear and BC dirs. - struct gkyl_range_iter iter; - gkyl_range_iter_init(&iter, &shearbc_perp_r); - while (gkyl_range_iter_next(&iter)) { - - int ic = 0; - for (int d=0; dlocal_bcdir_ext_r.ndim; d++) { - if (d != up->bc_dir && d != up->shear_dir) { - tar_idx[d] = iter.idx[ic]; - ic++; - } - } - tar_idx[up->bc_dir] = bc_dir_loc_tar; - - - struct gkyl_range_iter shear_dir_iter; - gkyl_range_iter_init(&shear_dir_iter, &up->shear_r); - while (gkyl_range_iter_next(&shear_dir_iter)) { - int shear_dir_idx = shear_dir_iter.idx[0]; - tar_idx[up->shear_dir] = shear_dir_idx; - - long loc = gkyl_range_idx(&up->local_bcdir_ext_r, tar_idx); - num_numcol_fidx_tar_ho[tar_count] = loc; - tar_count += 1; - } - } - - long *num_numcol_fidx_tar; - if (!up->use_gpu) { - num_numcol_fidx_tar = (long*) gkyl_malloc(up->fmat->num * up->fmat->nc * sizeof(long)); - memcpy(num_numcol_fidx_tar, num_numcol_fidx_tar_ho, up->fmat->num * up->fmat->nc * sizeof(long)); - } -#ifdef GKYL_HAVE_CUDA - if (up->use_gpu) { - num_numcol_fidx_tar = (long*) gkyl_cu_malloc(up->fmat->num * up->fmat->nc * sizeof(long)); - gkyl_cu_memcpy(num_numcol_fidx_tar, num_numcol_fidx_tar_ho, up->fmat->num * up->fmat->nc * sizeof(long), GKYL_CU_MEMCPY_H2D); - } -#endif - - gkyl_free(num_numcol_fidx_tar_ho); - - return num_numcol_fidx_tar; -} - -void -gkyl_bc_twistshift_choose_kernels(struct gkyl_basis basis, int cdim, int shift_poly_order, - struct gkyl_bc_twistshift_kernels *kers) -{ - int dim = basis.ndim; - int vdim = dim - cdim; - enum gkyl_basis_type basis_type = basis.b_type; - int poly_order = basis.poly_order; - switch (basis_type) { - case GKYL_BASIS_MODAL_GKHYBRID: - case GKYL_BASIS_MODAL_SERENDIPITY: - if (shift_poly_order == 1) { - kers->xlimdg = vdim==0? ser_twistshift_xlimdg_list_0v_yShp1[cdim-2].kernels[poly_order] - : ser_twistshift_xlimdg_list_2v_yShp1[cdim-2].kernels[poly_order]; - kers->ylimdg = vdim==0? ser_twistshift_ylimdg_list_0v_yShp1[cdim-2].kernels[poly_order] - : ser_twistshift_ylimdg_list_2v_yShp1[cdim-2].kernels[poly_order]; - kers->fullcell = vdim==0? ser_twistshift_fullcell_list_0v_yShp1[cdim-2].kernels[poly_order] - : ser_twistshift_fullcell_list_2v_yShp1[cdim-2].kernels[poly_order]; - } - else if (shift_poly_order == 2) { - assert(false); // MF 2025/09/20: removed 3x2v kernel because it's 8.5 MB. - kers->xlimdg = vdim==0? ser_twistshift_xlimdg_list_0v_yShp2[cdim-2].kernels[poly_order] - : ser_twistshift_xlimdg_list_2v_yShp2[cdim-2].kernels[poly_order]; - kers->ylimdg = vdim==0? ser_twistshift_ylimdg_list_0v_yShp2[cdim-2].kernels[poly_order] - : ser_twistshift_ylimdg_list_2v_yShp2[cdim-2].kernels[poly_order]; - kers->fullcell = vdim==0? ser_twistshift_fullcell_list_0v_yShp2[cdim-2].kernels[poly_order] - : ser_twistshift_fullcell_list_2v_yShp2[cdim-2].kernels[poly_order]; - } - return; - default: - assert(false); - break; - } + struct gkyl_dg_interpolate *interp = gkyl_dg_interpolate_new(1, &shift_b, + &grid_do, &grid_tar, &shear_r_do, shear_r_fine, (int[]) {0}, false); + gkyl_dg_interpolate_advance(interp, inp->shift_dg, shift_dg_fine); + gkyl_dg_interpolate_release(interp); } struct gkyl_bc_twistshift* gkyl_bc_twistshift_inew(const struct gkyl_bc_twistshift_inp *inp) { + struct gkyl_bc_twistshift *up = gkyl_malloc(sizeof(*up)); - // Allocate space for new updater. - struct gkyl_bc_twistshift *up = gkyl_malloc(sizeof(struct gkyl_bc_twistshift)); - - up->bc_dir = inp->bc_dir; - up->shift_dir = inp->shift_dir; - up->shear_dir = inp->shear_dir; - up->edge = inp->edge; - up->basis = *inp->basis; - up->grid = *inp->grid; up->use_gpu = inp->use_gpu; - up->local_bcdir_ext_r = *inp->bcdir_ext_update_r; - - // Assume the poly order of the DG shift is the same as that of the field, - // unless requested otherwise. - up->shift_poly_order = inp->basis->poly_order; - if (inp->shift_poly_order) - up->shift_poly_order = inp->shift_poly_order; - + up->filter_half_width = inp->filter_half_width; + up->filter_cutoff_wavelength = inp->filter_cutoff_wavelength; + up->upsample_factor = inp->upsample_factor > 1 ? inp->upsample_factor : 1; + + up->filter = NULL; + up->filt_buff = NULL; + up->ffine = NULL; + up->shift_dg_fine = NULL; + up->refine = NULL; + up->coarsen = NULL; + up->refine_func = bc_twistshift_refine_disabled; + up->coarsen_func = bc_twistshift_coarsen_disabled; + + // The half-width counts cells of the original grid. + up->half_width_fine = up->filter_half_width * up->upsample_factor; + + // A stencil one fine cell wide is the identity kernel. + assert(up->half_width_fine != 1); + // Supersampling is only useful if there is filtering. + if (up->upsample_factor > 1) + assert(up->filter_half_width > 0 && up->filter_cutoff_wavelength > 0.0); + + if (up->filter_half_width == 0) { + // Plain twist-shift. + struct gkyl_twistshift_dg_inp tsinp = { + .bc_dir = inp->bc_dir, + .shift_dir = inp->shift_dir, + .shear_dir = inp->shear_dir, + .edge = inp->edge, + .cdim = inp->cdim, + .bcdir_ext_update_r = inp->bcdir_ext_update_r, + .num_ghost = inp->num_ghost, + .basis = inp->basis, + .grid = inp->grid, + .shift_func = inp->shift_func, + .shift_func_ctx = inp->shift_func_ctx, + .shift_dg = inp->shift_dg, + .use_gpu = inp->use_gpu, + .shift_poly_order = inp->shift_poly_order, + }; + up->ts = gkyl_twistshift_dg_inew(&tsinp); + up->advance_func = bc_twistshift_advance_ts; + return up; + } + + // Upsampling and filtering attributes. const int ndim = inp->bcdir_ext_update_r->ndim; - // Check that it is being used for 3D or 5D. Likely only small changes are - // needed to make it work in other dimensions. - assert(ndim == 3 || ndim == 5); - - double lo1d[1], up1d[1]; int cells1d[1]; - - // Create 1D grid and range in the direction of the shear. - gkyl_range_init(&up->shear_r, 1, (int[]) {up->local_bcdir_ext_r.lower[inp->shear_dir]}, - (int[]) {up->local_bcdir_ext_r.upper[inp->shear_dir]}); - lo1d[0] = inp->grid->lower[up->shear_dir]; - up1d[0] = inp->grid->upper[up->shear_dir]; - cells1d[0] = inp->grid->cells[up->shear_dir]; - gkyl_rect_grid_init(&up->shear_grid, 1, lo1d, up1d, cells1d); - int idx[] = {up->shear_r.lower[0]}; - long linidx = gkyl_range_idx(&up->shear_r, idx); - - // Create 1D grid and range in the diretion of the shift. - gkyl_range_init(&up->shift_r, 1, (int[]) {up->local_bcdir_ext_r.lower[inp->shift_dir]}, - (int[]) {up->local_bcdir_ext_r.upper[inp->shift_dir]}); - lo1d[0] = inp->grid->lower[up->shift_dir]; - up1d[0] = inp->grid->upper[up->shift_dir]; - cells1d[0] = inp->grid->cells[up->shift_dir]; - gkyl_rect_grid_init(&up->shift_grid, 1, lo1d, up1d, cells1d); - - // Create 2D grid (and range) the twist-shift takes place in. - int dimlo, dimup; - if (up->shift_dir < up->shear_dir) { - dimlo = up->shift_dir; - dimup = up->shear_dir; - up->shift_dir_in_ts_grid = 0; - up->shear_dir_in_ts_grid = 1; - } - else { - dimlo = up->shear_dir; - dimup = up->shift_dir; - up->shift_dir_in_ts_grid = 1; - up->shear_dir_in_ts_grid = 0; - } - gkyl_range_init(&up->ts_r, 2, (int[]) {up->local_bcdir_ext_r.lower[dimlo], up->local_bcdir_ext_r.lower[dimup]}, - (int[]) {up->local_bcdir_ext_r.upper[dimlo], up->local_bcdir_ext_r.upper[dimup]}); - double lo2d[] = {inp->grid->lower[dimlo], inp->grid->lower[dimup]}; - double up2d[] = {inp->grid->upper[dimlo], inp->grid->upper[dimup]}; - int cells2d[] = {inp->grid->cells[dimlo], inp->grid->cells[dimup]}; - gkyl_rect_grid_init(&up->ts_grid, 2, lo2d, up2d, cells2d); - - // Project the shift onto the shift basis. - gkyl_cart_modal_serendip(&up->shift_b, 1, up->shift_poly_order); - if (inp->shift_func) { - up->shift_dg = gkyl_array_new(GKYL_DOUBLE, up->shift_b.num_basis, up->shear_r.volume); - gkyl_eval_on_nodes *evup = gkyl_eval_on_nodes_new(&up->shear_grid, &up->shift_b, 1, - inp->shift_func, inp->shift_func_ctx); - gkyl_eval_on_nodes_advance(evup, 0.0, &up->shear_r, up->shift_dg); - gkyl_eval_on_nodes_release(evup); - } - else { - up->shift_dg = gkyl_array_acquire(inp->shift_dg); - } - - // Function defining the shift (and its context). - if (shift_func_op == 0) { - up->shift_func = ts_shift_dg_eval; - up->shift_dg_eval_ctx.shift_dg = up->shift_dg; - up->shift_dg_eval_ctx.shift_b = &up->shift_b; - up->shift_dg_eval_ctx.shear_grid = &up->shear_grid; - up->shift_dg_eval_ctx.shear_r = &up->shear_r; - up->shift_func_ctx = &up->shift_dg_eval_ctx; - } - else if (shift_func_op == 1) { - up->shift_func = inp->shift_func; - up->shift_func_ctx = inp->shift_func_ctx; - } - else { - fprintf(stderr, "Twist-shift function option not recognized. Exiting...\n"); - assert(false); - } - - // Find the donor cells for each target. Store the number of donors for each - // shear_dir idx (num_do) & the shift_dir idx of each donor (shift_dir_idx_do). - // i.e. allocates and assigns num_do and up->shift_dir_idx_do. - ts_find_donors(up); - - // Array of cummulative number of donors at given shear_dir cell. - const int num_do_cum_sz = up->grid.cells[up->shear_dir]+1; - int num_do_cum_ho[num_do_cum_sz]; - for (int i=0; ishear_r.lower[0]; ishear_r.upper[0]+1; i++) - num_do_cum_ho[i] = num_do_cum_ho[i-1] + up->num_do[i-up->shear_r.lower[0]]; - - if (!up->use_gpu) { - up->num_do_cum = gkyl_malloc(num_do_cum_sz * sizeof(int)); - memcpy(up->num_do_cum, num_do_cum_ho, num_do_cum_sz * sizeof(int)); - } -#ifdef GKYL_HAVE_CUDA - if (up->use_gpu) { - up->num_do_cum = gkyl_cu_malloc(num_do_cum_sz * sizeof(int)); - gkyl_cu_memcpy(up->num_do_cum, num_do_cum_ho, num_do_cum_sz * sizeof(int), GKYL_CU_MEMCPY_H2D); - } -#endif - - // Choose the kernels that do the subcell and full cell integrals - up->kernels = gkyl_malloc(sizeof(struct gkyl_bc_twistshift_kernels)); - gkyl_bc_twistshift_choose_kernels(*inp->basis, inp->cdim, up->shift_poly_order, up->kernels); - - // The BC is applied as a set of matrix-matrix multiplications - // f_i = sum_{q}^{N_do(i)} A_q,i B_q,i - // where i indicates the shear_dir cell index, A_q is a - // num_basis x num_basis matrix containing the discretization - // of subcell integrals, B_q is a num_basis x (Ny * Nvpar * Nmu) matrix with - // the DG coefficients of f common to a given A_q matrix, and thus where f_i - // is a num_basis x (Ny*Nvpar*Nmu) matrix. - // - // Naming scheme: - // A_q: scimat (subscell integral matrices). - // B_q: fmat (distribution function matrices). - // A_q . B_q: mm_contr (contributions from mat-mat multiplication). - - // Calculate the entries in the matrices used to apply the BC. - up->scimat = ts_calc_mats(up); - - // Number of colums in fmat. - int fmat_num_col = 1; + // Ghost plane this BC fills, on the field's own grid. + if (inp->edge == GKYL_LOWER_EDGE) + gkyl_range_shorten_from_above(&up->coarse_ghost_r, inp->bcdir_ext_update_r, + inp->bc_dir, inp->num_ghost[inp->bc_dir]); + else + gkyl_range_shorten_from_below(&up->coarse_ghost_r, inp->bcdir_ext_update_r, + inp->bc_dir, inp->num_ghost[inp->bc_dir]); + + // Grid supersampled along shear_dir. + int fine_cells[GKYL_MAX_DIM]; + for (int d=0; dgrid->cells[d]; + fine_cells[inp->shear_dir] *= up->upsample_factor; + gkyl_rect_grid_init(&up->ts_grid, ndim, inp->grid->lower, inp->grid->upper, fine_cells); + // Range for supersampled cells. + int flo[GKYL_MAX_DIM], fup[GKYL_MAX_DIM]; for (int d=0; dbc_dir && d != up->shear_dir) - fmat_num_col *= up->local_bcdir_ext_r.upper[d] - up->local_bcdir_ext_r.lower[d] + 1; - } - - if (!up->use_gpu) { - up->fmat = gkyl_nmat_new(up->scimat->num, up->scimat->nr, fmat_num_col); - up->mm_contr = gkyl_nmat_new(up->scimat->num, up->scimat->nr, fmat_num_col); - } -#ifdef GKYL_HAVE_CUDA - if (up->use_gpu) { - up->fmat = gkyl_nmat_cu_dev_new(up->scimat->num, up->scimat->nr, fmat_num_col); - up->mm_contr = gkyl_nmat_cu_dev_new(up->scimat->num, up->scimat->nr, fmat_num_col); + flo[d] = up->coarse_ghost_r.lower[d]; + fup[d] = up->coarse_ghost_r.upper[d]; } -#endif + flo[inp->shear_dir] = (flo[inp->shear_dir]-1)*up->upsample_factor + 1; + fup[inp->shear_dir] = fup[inp->shear_dir]*up->upsample_factor; + gkyl_range_init(&up->ts_ext_r, ndim, flo, fup); + gkyl_sub_range_init(&up->ts_update_r, &up->ts_ext_r, flo, fup); - // Index translation from num-numcol plane index to linear index into the - // donor distribution function gkyl_array. - up->num_numcol_fidx_do = ts_calc_num_numcol_fidx_do(up); + up->ffine = mkarr(inp->use_gpu, inp->basis->num_basis, up->ts_ext_r.volume); + up->filt_buff = mkarr(inp->use_gpu, inp->basis->num_basis, up->ts_ext_r.volume); - // Index translation from num-numcol plane index to linear index into the - // tar distribution function gkyl_array. - up->num_numcol_fidx_tar = ts_calc_num_numcol_fidx_tar(up); - - // Permutted ghost range, for indexing into the target field. - // Order: Shift direction, redundant directions, shear direction. - int lo4D[ndim-1], up4D[ndim-1]; - lo4D[0] = up->local_bcdir_ext_r.lower[up->shift_dir]; - up4D[0] = up->local_bcdir_ext_r.upper[up->shift_dir]; - int ic = 1; - for (int d=0; dbc_dir && d != up->shear_dir && d != up->shift_dir) { - lo4D[ic] = up->local_bcdir_ext_r.lower[d]; - up4D[ic] = up->local_bcdir_ext_r.upper[d]; - ic++; - } - } - lo4D[ndim-2] = up->local_bcdir_ext_r.lower[up->shear_dir]; - up4D[ndim-2] = up->local_bcdir_ext_r.upper[up->shear_dir]; - gkyl_range_init(&up->permutted_ghost_r, ndim-1, lo4D, up4D); - - // Create a ghost range, to clear it before adding contributions from TS BC. + // Ghost plane on the supersampled grid. if (inp->edge == GKYL_LOWER_EDGE) - gkyl_range_shorten_from_above(&up->ghost_r, &up->local_bcdir_ext_r, inp->bc_dir, inp->num_ghost[inp->bc_dir]); + gkyl_range_shorten_from_above(&up->ghost_r, &up->ts_update_r, inp->bc_dir, + inp->num_ghost[inp->bc_dir]); else - gkyl_range_shorten_from_below(&up->ghost_r, &up->local_bcdir_ext_r, inp->bc_dir, inp->num_ghost[inp->bc_dir]); + gkyl_range_shorten_from_below(&up->ghost_r, &up->ts_update_r, inp->bc_dir, + inp->num_ghost[inp->bc_dir]); + + up->filter = gkyl_dg_lowpass_filter_new(inp->shear_dir, up->half_width_fine, + up->filter_cutoff_wavelength, inp->basis, &up->ts_grid, &up->ghost_r, inp->use_gpu); + + if (up->upsample_factor > 1) { + up->refine = gkyl_dg_interpolate_new(inp->cdim, inp->basis, inp->grid, &up->ts_grid, + &up->coarse_ghost_r, &up->ghost_r, inp->num_ghost, inp->use_gpu); + up->coarsen = gkyl_dg_interpolate_new(inp->cdim, inp->basis, &up->ts_grid, inp->grid, + &up->ghost_r, &up->coarse_ghost_r, inp->num_ghost, inp->use_gpu); + up->refine_func = bc_twistshift_refine_enabled; + up->coarsen_func = bc_twistshift_coarsen_enabled; + } + + // Need to upsample the DG shift too. + struct gkyl_array *shift_dg = inp->shift_dg; + if (inp->shift_dg && up->upsample_factor > 1) { + struct gkyl_range shear_r_fine; + gkyl_range_init(&shear_r_fine, 1, (int[]) {flo[inp->shear_dir]}, + (int[]) {fup[inp->shear_dir]}); + up->shift_dg_fine = gkyl_array_new(GKYL_DOUBLE, inp->shift_dg->ncomp, shear_r_fine.volume); + bc_twistshift_refine_shift(inp, &up->ts_grid, &shear_r_fine, up->shift_dg_fine); + shift_dg = up->shift_dg_fine; + } + + struct gkyl_twistshift_dg_inp tsinp = { + .bc_dir = inp->bc_dir, + .shift_dir = inp->shift_dir, + .shear_dir = inp->shear_dir, + .edge = inp->edge, + .cdim = inp->cdim, + .bcdir_ext_update_r = &up->ts_update_r, + .num_ghost = inp->num_ghost, + .basis = inp->basis, + .grid = &up->ts_grid, + .shift_func = inp->shift_func, + .shift_func_ctx = inp->shift_func_ctx, + .shift_dg = shift_dg, + .use_gpu = inp->use_gpu, + .shift_poly_order = inp->shift_poly_order, + }; + up->ts = gkyl_twistshift_dg_inew(&tsinp); + up->advance_func = bc_twistshift_advance_ts_filtered; return up; } @@ -1902,23 +226,27 @@ struct gkyl_bc_twistshift* gkyl_bc_twistshift_new(int bc_dir, int shift_dir, int shear_dir, enum gkyl_edge_loc edge, int cdim, const struct gkyl_range *bcdir_ext_update_r, const int *num_ghost, const struct gkyl_basis *basis, const struct gkyl_rect_grid *grid, evalf_t shift_func, void *shift_func_ctx, - struct gkyl_array *shift_dg, int shift_poly_order, bool use_gpu) + struct gkyl_array *shift_dg, int shift_poly_order, int filter_half_width, + double filter_cutoff_wavelength, int upsample_factor, bool use_gpu) { struct gkyl_bc_twistshift_inp inp = { - .bc_dir = bc_dir , - .shift_dir = shift_dir , - .shear_dir = shear_dir , - .edge = edge , - .cdim = cdim , - .bcdir_ext_update_r = bcdir_ext_update_r, - .num_ghost = num_ghost , - .basis = basis , - .grid = grid , - .shift_func = shift_func , - .shift_func_ctx = shift_func_ctx , - .shift_dg = shift_dg , - .use_gpu = use_gpu , - .shift_poly_order = shift_poly_order , + .bc_dir = bc_dir , + .shift_dir = shift_dir , + .shear_dir = shear_dir , + .edge = edge , + .cdim = cdim , + .bcdir_ext_update_r = bcdir_ext_update_r , + .num_ghost = num_ghost , + .basis = basis , + .grid = grid , + .shift_func = shift_func , + .shift_func_ctx = shift_func_ctx , + .shift_dg = shift_dg , + .use_gpu = use_gpu , + .shift_poly_order = shift_poly_order , + .filter_half_width = filter_half_width , + .filter_cutoff_wavelength = filter_cutoff_wavelength, + .upsample_factor = upsample_factor , }; return gkyl_bc_twistshift_inew(&inp); } @@ -1926,113 +254,23 @@ gkyl_bc_twistshift_new(int bc_dir, int shift_dir, int shear_dir, void gkyl_bc_twistshift_advance(struct gkyl_bc_twistshift *up, struct gkyl_array *fdo, struct gkyl_array *ftar) { - -#ifdef GKYL_HAVE_CUDA - if (up->use_gpu) { - gkyl_bc_twistshift_advance_cu(up, fdo, ftar); - return; - } -#endif - - // Assign the distribution matrices. - // This assumes that fdo->ncomp = fmat->nr. - // Recall: - // fmat->num = sum_i^Nx num_do(i) - // fmat->nr = num_basis = ncomp - // fmat->nc = Ny*Nvpar*Nmu - for (size_t i=0; ifmat->num * up->fmat->nr * up->fmat->nc; i++) { - - long nc_idx = i / up->fmat->nr; // num-num_col index: current num-num_col plane. - int row_idx = i % up->fmat->nr; // row index: current DG coeff. - - // This if-statement may only be needed in GPU kernel, not for CPUs. - if ((nc_idx < up->fmat->num * up->fmat->nc) && (row_idx < fdo->ncomp)) { - const double *fdo_c = (const double*) gkyl_array_cfetch(fdo, up->num_numcol_fidx_do[nc_idx]); - struct gkyl_mat mcurr = gkyl_nmat_get(up->fmat, nc_idx % up->fmat->num); - - gkyl_mat_set(&mcurr, row_idx, nc_idx/up->fmat->num, fdo_c[row_idx]); - } - } - - // Perform the mat-mat multiplications. - gkyl_nmat_mm(1.0, 0.0, GKYL_NO_TRANS, up->scimat, GKYL_NO_TRANS, up->fmat, up->mm_contr); - - // Clear the ghost range. - gkyl_array_clear_range(ftar, 0.0, &up->ghost_r); - - // Perform reduction over num_do contributions from mat-mat mults (mm_contr). - int num_cells_skin = (up->shear_r.upper[0]-up->shear_r.lower[0]+1) * up->fmat->nc; - for (size_t i=0; incomp * num_cells_skin; i++) { - - long linidx_tar = i / ftar->ncomp; - int row_idx = i % ftar->ncomp; - - // This if-statement may only be needed in GPU kernel, not for CPUs. - if ((linidx_tar < num_cells_skin) && (row_idx < ftar->ncomp)) { - double *ftar_c = (double*) gkyl_array_fetch(ftar, up->num_numcol_fidx_tar[linidx_tar]); - - int idx[GKYL_MAX_DIM] = {1}; - gkyl_sub_range_inv_idx(&up->permutted_ghost_r, linidx_tar, idx); - - int ac[GKYL_MAX_DIM] = {1}; - for (int d=2; dgrid.ndim-1; d++) - ac[d-2] = up->grid.cells[d+1]; - ac[up->permutted_ghost_r.ndim-2] = up->mm_contr->num; - - int start = 0; - for (int d=0; dpermutted_ghost_r.ndim-1; d++) - start = (start + (idx[d]-1)) * ac[d]; - - int shear_idx = idx[up->permutted_ghost_r.ndim-1]; - - int do_start = up->num_do_cum[shear_idx-1]; - int do_end = up->num_do_cum[shear_idx-1+1]; - for (int j=do_start; jmm_contr, linidx_mm_contr % up->mm_contr->num); - ftar_c[row_idx] += gkyl_mat_get(&mat, row_idx, linidx_mm_contr / up->mm_contr->num); - } - } - } + up->advance_func(up, fdo, ftar); } -struct gkyl_array* -gkyl_bc_twistshift_get_shift_objects(struct gkyl_bc_twistshift *up, struct gkyl_rect_grid *shear_grid, - struct gkyl_range *shear_r, struct gkyl_basis *shift_b) -{ - *shear_grid = up->shear_grid; - *shear_r = up->shear_r ; - *shift_b = up->shift_b ; - return gkyl_array_acquire(up->shift_dg); -}; - void -gkyl_bc_twistshift_release(struct gkyl_bc_twistshift *up) { - // Release memory associated with this updater. - if (!up->use_gpu) { - gkyl_free(up->num_do_cum); - gkyl_free(up->num_numcol_fidx_do); - gkyl_free(up->num_numcol_fidx_tar); +gkyl_bc_twistshift_release(struct gkyl_bc_twistshift *up) +{ + gkyl_twistshift_dg_release(up->ts); + if (up->filter) { + gkyl_dg_lowpass_filter_release(up->filter); + gkyl_array_release(up->ffine); + gkyl_array_release(up->filt_buff); } -#ifdef GKYL_HAVE_CUDA - if (up->use_gpu) { - gkyl_cu_free(up->num_do_cum); - gkyl_cu_free(up->num_numcol_fidx_do); - gkyl_cu_free(up->num_numcol_fidx_tar); + if (up->shift_dg_fine) + gkyl_array_release(up->shift_dg_fine); + if (up->refine) { + gkyl_dg_interpolate_release(up->refine); + gkyl_dg_interpolate_release(up->coarsen); } -#endif - - gkyl_nmat_release(up->fmat); - gkyl_nmat_release(up->mm_contr); - - gkyl_nmat_release(up->scimat); - - gkyl_free(up->kernels); - - gkyl_array_release(up->shift_dg); - - gkyl_free(up->num_do); - gkyl_free(up->shift_dir_idx_do); - gkyl_free(up); } diff --git a/gyrokinetic/zero/gkyl_bc_twistshift.h b/gyrokinetic/zero/gkyl_bc_twistshift.h index cb6ccec89d..73c7952d99 100644 --- a/gyrokinetic/zero/gkyl_bc_twistshift.h +++ b/gyrokinetic/zero/gkyl_bc_twistshift.h @@ -7,14 +7,14 @@ #include #include -// Object type +// Object type. typedef struct gkyl_bc_twistshift gkyl_bc_twistshift; struct gkyl_bc_twistshift_inp { int bc_dir; // Direction in which to apply this BC. int shift_dir; // Direction of the shift. int shear_dir; // Direction in which the shift varies (shear). - enum gkyl_edge_loc edge; // Edge of to apply this BC at (lower/upper). + enum gkyl_edge_loc edge; // Edge to apply this BC at (lower/upper). int cdim; // Configuration space dimensions. const struct gkyl_range *bcdir_ext_update_r; // Local range where to apply BC, extended in bc_dir. const int *num_ghost; // Number of ghost cells in each direction. @@ -26,23 +26,29 @@ struct gkyl_bc_twistshift_inp { bool use_gpu; // Whether to apply the BC using the GPU. // Optional inputs: int shift_poly_order; // Basis order for the DG representation of the shift. + int filter_half_width; // Filter stencil half-width M in cells of grid (0 = no filter). + double filter_cutoff_wavelength; // Filter cutoff wavelength (physical units). + int upsample_factor; // Supersampling factor along shear_dir (0/1 = none). }; /** - * Create a new updater to apply twist-shift BCs. + * Create a new updater to apply the twist-shift BC, optionally with + * supersampling and low-pass filtering to de-alias the shifted field. It + * combines a gkyl_twistshift_dg updater with, when requested, a + * gkyl_dg_lowpass_filter and gkyl_dg_interpolate operators. * * @param inp bc_twistshift_inp struct containing the inputs to the updater. * @return New updater pointer. */ struct gkyl_bc_twistshift* gkyl_bc_twistshift_inew(const struct gkyl_bc_twistshift_inp *inp); - + /** * Create a new updater to apply twist-shift BCs, passing each argument separately. * * @param bc_dir Direction in which to apply this BC. * @param shift_dir Direction of the shift. * @param shear_dir Direction in which the shift varies (shear). - * @param edge Edge of to apply this BC at (lower/upper). + * @param edge Edge to apply this BC at (lower/upper). * @param cdim Configuration space dimensions. * @param bcdir_ext_update_r Local range where to apply BC, extended in bc_dir. * @param num_ghost Number of ghost cells in each direction. @@ -52,17 +58,21 @@ struct gkyl_bc_twistshift* gkyl_bc_twistshift_inew(const struct gkyl_bc_twistshi * @param shift_func_ctx Context for shift_func. * @param shift_dg Discretized shift. * @param shift_poly_order Basis order for the DG representation of the shift (optional). + * @param filter_half_width Filter stencil half-width M in cells of grid (0 = no filter). + * @param filter_cutoff_wavelength Filter cutoff wavelength (physical units). + * @param upsample_factor Supersampling factor along shear_dir (0/1 = none). * @param use_gpu Whether to apply the BC using the GPU. * @return New updater pointer. */ struct gkyl_bc_twistshift* gkyl_bc_twistshift_new(int bc_dir, int shift_dir, int shear_dir, enum gkyl_edge_loc edge, int cdim, const struct gkyl_range *bcdir_ext_update_r, const int *num_ghost, const struct gkyl_basis *basis, const struct gkyl_rect_grid *grid, evalf_t shift_func, void *shift_func_ctx, - struct gkyl_array *shift_dg, int shift_poly_order, bool use_gpu); + struct gkyl_array *shift_dg, int shift_poly_order, int filter_half_width, + double filter_cutoff_wavelength, int upsample_factor, bool use_gpu); /** - * Apply the twist-shift. It assumes that periodicity along bc_dir has been - * applied to the donor field. Can be used in-place. + * Apply the twist-shift periodic BC. Expects periodicity along bc_dir to have + * been applied to the donor field beforehand. Can be used in-place. * * @param up Twist-shift BC updater object. * @param fdo Donor field. @@ -71,21 +81,7 @@ struct gkyl_bc_twistshift* gkyl_bc_twistshift_new(int bc_dir, int shift_dir, int void gkyl_bc_twistshift_advance(struct gkyl_bc_twistshift *up, struct gkyl_array *fdo, struct gkyl_array *ftar); /** - * Return pointers to the discretized shift, its range, and grid and basis. - * - * Pointer to shift_dg needs to be released with gkyl_array_release. - * - * @param up Twist-shift BC updater object. - * @param shear_grid Grid on which shift is defined. - * @param shear_r Range for the shift. - * @param shift_b Basis shift_dg coefficients are expanded on. - * @return Discretized shift. - */ -struct gkyl_array* gkyl_bc_twistshift_get_shift_objects(struct gkyl_bc_twistshift *up, - struct gkyl_rect_grid *shear_grid, struct gkyl_range *shear_r, struct gkyl_basis *shift_b); - -/** - * Free memory associated with bc_twistshift updater. + * Free memory associated with the bc_twistshift updater. * * @param up BC updater. */ diff --git a/gyrokinetic/zero/gkyl_bc_twistshift_priv.h b/gyrokinetic/zero/gkyl_bc_twistshift_priv.h index f66ffe186d..3b39317cfc 100644 --- a/gyrokinetic/zero/gkyl_bc_twistshift_priv.h +++ b/gyrokinetic/zero/gkyl_bc_twistshift_priv.h @@ -1,165 +1,41 @@ #pragma once -// Private header for bc_twistshift updater, not for direct use in user code. +// Private header for the bc_twistshift orchestrator, not for direct use in +// user code. #include -#include -#include -#include -#include -#include -#include // memcpy - -// Function pointer type for twistshift kernels. -typedef void (*twistshift_xlimdg_t)(double sFac, const double *xLimLo, - const double *xLimUp, double yLimLo, double yLimUp, - double dyDo, double yOff, const double *ySh, struct gkyl_mat *tsmat); - -typedef void (*twistshift_ylimdg_t)(double sFac, double xLimLo, - double xLimUp, const double *yLimLo, const double *yLimUp, - double dyDo, double yOff, const double *ySh, struct gkyl_mat *tsmat); - -typedef void (*twistshift_fullcell_t)(double dyDo, double yOff, - const double *ySh, struct gkyl_mat *tsmat); - -typedef struct { twistshift_xlimdg_t kernels[3]; } twistshift_xlimdg_kern_list; // For use in kernel tables. -typedef struct { twistshift_ylimdg_t kernels[3]; } twistshift_ylimdg_kern_list; // For use in kernel tables. -typedef struct { twistshift_fullcell_t kernels[3]; } twistshift_fullcell_kern_list; // For use in kernel tables. - -// Serendipity kernels. -// p=1 representation of the shift: -static const twistshift_xlimdg_kern_list ser_twistshift_xlimdg_list_0v_yShp1[] = { - {NULL, twistshift_xlimdg_2x_ser_p1_yshift_p1, NULL,}, - {NULL, twistshift_xlimdg_3x_ser_p1_yshift_p1, NULL,}, -}; -static const twistshift_ylimdg_kern_list ser_twistshift_ylimdg_list_0v_yShp1[] = { - {NULL, twistshift_ylimdg_2x_ser_p1_yshift_p1, NULL,}, - {NULL, twistshift_ylimdg_3x_ser_p1_yshift_p1, NULL,}, -}; -static const twistshift_fullcell_kern_list ser_twistshift_fullcell_list_0v_yShp1[] = { - {NULL, twistshift_fullcell_2x_ser_p1_yshift_p1, NULL,}, - {NULL, twistshift_fullcell_3x_ser_p1_yshift_p1, NULL,}, -}; - -static const twistshift_xlimdg_kern_list ser_twistshift_xlimdg_list_2v_yShp1[] = { - {NULL, NULL, NULL,}, - {NULL, twistshift_xlimdg_3x2v_ser_p1_yshift_p1, NULL,}, -}; -static const twistshift_ylimdg_kern_list ser_twistshift_ylimdg_list_2v_yShp1[] = { - {NULL, NULL, NULL,}, - {NULL, twistshift_ylimdg_3x2v_ser_p1_yshift_p1, NULL,}, -}; -static const twistshift_fullcell_kern_list ser_twistshift_fullcell_list_2v_yShp1[] = { - {NULL, NULL, NULL,}, - {NULL, twistshift_fullcell_3x2v_ser_p1_yshift_p1, NULL,}, -}; -// p=2 representation of the shift: -static const twistshift_xlimdg_kern_list ser_twistshift_xlimdg_list_0v_yShp2[] = { - {NULL, twistshift_xlimdg_2x_ser_p1_yshift_p2, NULL,}, - {NULL, twistshift_xlimdg_3x_ser_p1_yshift_p2, NULL,}, -}; -static const twistshift_ylimdg_kern_list ser_twistshift_ylimdg_list_0v_yShp2[] = { - {NULL, twistshift_ylimdg_2x_ser_p1_yshift_p2, NULL,}, - {NULL, twistshift_ylimdg_3x_ser_p1_yshift_p2, NULL,}, -}; -static const twistshift_fullcell_kern_list ser_twistshift_fullcell_list_0v_yShp2[] = { - {NULL, twistshift_fullcell_2x_ser_p1_yshift_p2, NULL,}, - {NULL, twistshift_fullcell_3x_ser_p1_yshift_p2, NULL,}, -}; - -static const twistshift_xlimdg_kern_list ser_twistshift_xlimdg_list_2v_yShp2[] = { - {NULL, NULL, NULL,}, - {NULL, NULL, NULL,}, -}; -static const twistshift_ylimdg_kern_list ser_twistshift_ylimdg_list_2v_yShp2[] = { - {NULL, NULL, NULL,}, - {NULL, NULL, NULL,}, -}; -static const twistshift_fullcell_kern_list ser_twistshift_fullcell_list_2v_yShp2[] = { - {NULL, NULL, NULL,}, - {NULL, NULL, NULL,}, -}; - - -struct gkyl_bc_twistshift_kernels { - twistshift_xlimdg_t xlimdg; - twistshift_ylimdg_t ylimdg; - twistshift_fullcell_t fullcell; -}; - -struct ts_shift_dg_eval_ctx { - struct gkyl_array *shift_dg; // DG representation of the shift function. - struct gkyl_basis *shift_b; // Basis for the shift. - struct gkyl_rect_grid *shear_grid; // shear grid along x. - struct gkyl_range *shear_r; // Shear grid range. -}; +#include +#include +#include +#include +#include // Primary struct in this updater. struct gkyl_bc_twistshift { - int bc_dir; // Direction of the BC is applied in. - int shift_dir; // Direction of the shift. - int shear_dir; // Direction the shift varies in (shear). - enum gkyl_edge_loc edge; // Indicates if BC is for lowe/upper edge. - struct gkyl_basis basis; // Basis the shifted field is defined with. - struct gkyl_range local_bcdir_ext_r; // Local range. - struct gkyl_rect_grid grid; // Grid the shifted field is defined in. - evalf_t shift_func; // Function defining the shift. - void *shift_func_ctx; // Context for shift_func. - struct ts_shift_dg_eval_ctx shift_dg_eval_ctx; // Context for DG shift_func. bool use_gpu; // Whether to apply the BC on the GPU. - struct gkyl_rect_grid shift_grid; // 1D grid in the direction of the shift. - struct gkyl_range shift_r; // 1D range in the direction of the shift. + struct gkyl_twistshift_dg *ts; // Pure twist-shift updater. - struct gkyl_rect_grid shear_grid; // 1D grid in the direction of the shear. - struct gkyl_range shear_r; // 1D range in the direction of the shear. + void (*advance_func)(struct gkyl_bc_twistshift *up, struct gkyl_array *fdo, struct gkyl_array *ftar); - struct gkyl_rect_grid ts_grid; // Grid the shift twistshift takes place in. - struct gkyl_range ts_r; // Range the twistshift takes place in. - int shift_dir_in_ts_grid; // Dimension the shift is in, in the TS grid. - int shear_dir_in_ts_grid; // Dimension the shear is in, in the TS grid. + int filter_half_width; // Filter stencil half-width M in cells of the field's + // own grid (0 = no filter). + int half_width_fine; // The same stencil measured in supersampled cells. + double filter_cutoff_wavelength; // Filter cutoff wavelength. + int upsample_factor; // Supersampling factor along shear_dir. - int shift_poly_order; // Poly order of the DG representation of the shift. - struct gkyl_basis shift_b; // 1D Basis for the DG shift. - struct gkyl_array *shift_dg; // DG shift. + struct gkyl_dg_lowpass_filter *filter; // Post-shift filter along shear_dir. + struct gkyl_rect_grid ts_grid; // Grid refined along shear_dir. + struct gkyl_range ts_ext_r; // ffine and filt_buff range. + struct gkyl_range ts_update_r; // Update range on ts_grid. + struct gkyl_range ghost_r; // Ghost plane the twist-shift fills, on ts_grid. + struct gkyl_range coarse_ghost_r; // Same plane on the field's own grid. + struct gkyl_array *ffine; // Ghost plane on the refined grid. + struct gkyl_array *filt_buff; // Buffer for the filter (shaped like ffine). + struct gkyl_array *shift_dg_fine; // Input shift refined onto the fine shear grid. - int *num_do; // Number of donors at each cell in shear_dir; - int *shift_dir_idx_do; // Indices of donor cells, in the direction of the - // shift, for each cell in the TS grid. - - struct gkyl_bc_twistshift_kernels *kernels; // kernels for sub-cell integrals. - - // Projection object used in constructing the matrices. - struct gkyl_eval_on_nodes *ev_on_nod1d; - // Evaluations of a function at 1D nodes. - struct gkyl_array *func_nod1d; - - struct gkyl_nmat *scimat; // Subcell integral matrices. - struct gkyl_nmat *fmat; // Distribution function matrices. - struct gkyl_nmat *mm_contr; // Contribution resulting from a mat-mat mult. - - long *num_numcol_fidx_do; // 1D indexer, from a index identitying the num-numcol - // plane (in the num-numcol-num_basis space), to a - // linear index into the donor distribution function f. - - long *num_numcol_fidx_tar; // 1D indexer, from a index identitying the num-numcol - // plane (in the num-numcol-num_basis space), to a - // linear index into the target distribution function f. - - int *num_do_cum; // Cumulative number of donors up to a give cell in shear_dir; - struct gkyl_range permutted_ghost_r; // Ghost range to populate in the target - // field, with some dimensions permutted. - struct gkyl_range ghost_r; // Ghost range this BC fills. + struct gkyl_dg_interpolate *refine; // Coarse ghost plane -> fine ghost plane. + struct gkyl_dg_interpolate *coarsen; // Fine ghost plane -> coarse ghost plane. + void (*refine_func)(struct gkyl_bc_twistshift *up, struct gkyl_array *fdo); + void (*coarsen_func)(struct gkyl_bc_twistshift *up, struct gkyl_array *ftar); }; - -#ifdef GKYL_HAVE_CUDA -/** - * Apply the twist-shift on the NVIDIA GPU. It assumes that periodicity along bc_dir has been - * applied to the donor field. Can be used in-place. - * - * @param up Twist-shift BC updater object. - * @param fdo Donor field. - * @param ftar Target field. - */ -void gkyl_bc_twistshift_advance_cu(struct gkyl_bc_twistshift *up, struct gkyl_array *fdo, struct gkyl_array *ftar); -#endif diff --git a/gyrokinetic/zero/gkyl_twistshift_dg.h b/gyrokinetic/zero/gkyl_twistshift_dg.h new file mode 100644 index 0000000000..084bad037d --- /dev/null +++ b/gyrokinetic/zero/gkyl_twistshift_dg.h @@ -0,0 +1,92 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +// Object type +typedef struct gkyl_twistshift_dg gkyl_twistshift_dg; + +struct gkyl_twistshift_dg_inp { + int bc_dir; // Direction in which to apply this BC. + int shift_dir; // Direction of the shift. + int shear_dir; // Direction in which the shift varies (shear). + enum gkyl_edge_loc edge; // Edge of to apply this BC at (lower/upper). + int cdim; // Configuration space dimensions. + const struct gkyl_range *bcdir_ext_update_r; // Local range where to apply BC, extended in bc_dir. + const int *num_ghost; // Number of ghost cells in each direction. + const struct gkyl_basis *basis; // Basis of the field shifted. + const struct gkyl_rect_grid *grid; // Grid the field shifted is defined on. + evalf_t shift_func; // Function defining the shift. + void *shift_func_ctx; // Context for shift_func. + struct gkyl_array *shift_dg; // Discretized shift. + bool use_gpu; // Whether to apply the BC using the GPU. + // Optional inputs: + int shift_poly_order; // Basis order for the DG representation of the shift. +}; + +/** + * Create a new updater to apply twist-shift BCs. + * + * @param inp twistshift_dg_inp struct containing the inputs to the updater. + * @return New updater pointer. + */ +struct gkyl_twistshift_dg* gkyl_twistshift_dg_inew(const struct gkyl_twistshift_dg_inp *inp); + +/** + * Create a new updater to apply twist-shift BCs, passing each argument separately. + * + * @param bc_dir Direction in which to apply this BC. + * @param shift_dir Direction of the shift. + * @param shear_dir Direction in which the shift varies (shear). + * @param edge Edge to apply this BC at (lower/upper). + * @param cdim Configuration space dimensions. + * @param bcdir_ext_update_r Local range where to apply BC, extended in bc_dir. + * @param num_ghost Number of ghost cells in each direction. + * @param basis Basis of the field shifted. + * @param grid Grid the field shifted is defined on. + * @param shift_func Function defining the shift. + * @param shift_func_ctx Context for shift_func. + * @param shift_dg Discretized shift. + * @param shift_poly_order Basis order for the DG representation of the shift (optional). + * @param use_gpu Whether to apply the BC using the GPU. + * @return New updater pointer. + */ +struct gkyl_twistshift_dg* gkyl_twistshift_dg_new(int bc_dir, int shift_dir, int shear_dir, + enum gkyl_edge_loc edge, int cdim, const struct gkyl_range *bcdir_ext_update_r, const int *num_ghost, + const struct gkyl_basis *basis, const struct gkyl_rect_grid *grid, evalf_t shift_func, void *shift_func_ctx, + struct gkyl_array *shift_dg, int shift_poly_order, bool use_gpu); + +/** + * Apply the twist-shift. It assumes that periodicity along bc_dir has been + * applied to the donor field. Can be used in-place. + * + * @param up Twist-shift BC updater object. + * @param fdo Donor field. + * @param ftar Target field. + */ +void gkyl_twistshift_dg_advance(struct gkyl_twistshift_dg *up, struct gkyl_array *fdo, struct gkyl_array *ftar); + +/** + * Return pointers to the discretized shift, its range, and grid and basis. + * + * Pointer to shift_dg needs to be released with gkyl_array_release. + * + * @param up Twist-shift BC updater object. + * @param shear_grid Grid on which shift is defined. + * @param shear_r Range for the shift. + * @param shift_b Basis shift_dg coefficients are expanded on. + * @return Discretized shift. + */ +struct gkyl_array* gkyl_twistshift_dg_get_shift_objects(struct gkyl_twistshift_dg *up, + struct gkyl_rect_grid *shear_grid, struct gkyl_range *shear_r, struct gkyl_basis *shift_b); + +/** + * Free memory associated with twistshift_dg updater. + * + * @param up BC updater. + */ +void gkyl_twistshift_dg_release(struct gkyl_twistshift_dg *up); diff --git a/gyrokinetic/zero/gkyl_twistshift_dg_priv.h b/gyrokinetic/zero/gkyl_twistshift_dg_priv.h new file mode 100644 index 0000000000..d927f0edea --- /dev/null +++ b/gyrokinetic/zero/gkyl_twistshift_dg_priv.h @@ -0,0 +1,165 @@ +#pragma once + +// Private header for twistshift_dg updater, not for direct use in user code. + +#include +#include +#include +#include +#include +#include +#include // memcpy + +// Function pointer type for twistshift kernels. +typedef void (*twistshift_xlimdg_t)(double sFac, const double *xLimLo, + const double *xLimUp, double yLimLo, double yLimUp, + double dyDo, double yOff, const double *ySh, struct gkyl_mat *tsmat); + +typedef void (*twistshift_ylimdg_t)(double sFac, double xLimLo, + double xLimUp, const double *yLimLo, const double *yLimUp, + double dyDo, double yOff, const double *ySh, struct gkyl_mat *tsmat); + +typedef void (*twistshift_fullcell_t)(double dyDo, double yOff, + const double *ySh, struct gkyl_mat *tsmat); + +typedef struct { twistshift_xlimdg_t kernels[3]; } twistshift_xlimdg_kern_list; // For use in kernel tables. +typedef struct { twistshift_ylimdg_t kernels[3]; } twistshift_ylimdg_kern_list; // For use in kernel tables. +typedef struct { twistshift_fullcell_t kernels[3]; } twistshift_fullcell_kern_list; // For use in kernel tables. + +// Serendipity kernels. +// p=1 representation of the shift: +static const twistshift_xlimdg_kern_list ser_twistshift_xlimdg_list_0v_yShp1[] = { + {NULL, twistshift_xlimdg_2x_ser_p1_yshift_p1, NULL,}, + {NULL, twistshift_xlimdg_3x_ser_p1_yshift_p1, NULL,}, +}; +static const twistshift_ylimdg_kern_list ser_twistshift_ylimdg_list_0v_yShp1[] = { + {NULL, twistshift_ylimdg_2x_ser_p1_yshift_p1, NULL,}, + {NULL, twistshift_ylimdg_3x_ser_p1_yshift_p1, NULL,}, +}; +static const twistshift_fullcell_kern_list ser_twistshift_fullcell_list_0v_yShp1[] = { + {NULL, twistshift_fullcell_2x_ser_p1_yshift_p1, NULL,}, + {NULL, twistshift_fullcell_3x_ser_p1_yshift_p1, NULL,}, +}; + +static const twistshift_xlimdg_kern_list ser_twistshift_xlimdg_list_2v_yShp1[] = { + {NULL, NULL, NULL,}, + {NULL, twistshift_xlimdg_3x2v_ser_p1_yshift_p1, NULL,}, +}; +static const twistshift_ylimdg_kern_list ser_twistshift_ylimdg_list_2v_yShp1[] = { + {NULL, NULL, NULL,}, + {NULL, twistshift_ylimdg_3x2v_ser_p1_yshift_p1, NULL,}, +}; +static const twistshift_fullcell_kern_list ser_twistshift_fullcell_list_2v_yShp1[] = { + {NULL, NULL, NULL,}, + {NULL, twistshift_fullcell_3x2v_ser_p1_yshift_p1, NULL,}, +}; +// p=2 representation of the shift: +static const twistshift_xlimdg_kern_list ser_twistshift_xlimdg_list_0v_yShp2[] = { + {NULL, twistshift_xlimdg_2x_ser_p1_yshift_p2, NULL,}, + {NULL, twistshift_xlimdg_3x_ser_p1_yshift_p2, NULL,}, +}; +static const twistshift_ylimdg_kern_list ser_twistshift_ylimdg_list_0v_yShp2[] = { + {NULL, twistshift_ylimdg_2x_ser_p1_yshift_p2, NULL,}, + {NULL, twistshift_ylimdg_3x_ser_p1_yshift_p2, NULL,}, +}; +static const twistshift_fullcell_kern_list ser_twistshift_fullcell_list_0v_yShp2[] = { + {NULL, twistshift_fullcell_2x_ser_p1_yshift_p2, NULL,}, + {NULL, twistshift_fullcell_3x_ser_p1_yshift_p2, NULL,}, +}; + +static const twistshift_xlimdg_kern_list ser_twistshift_xlimdg_list_2v_yShp2[] = { + {NULL, NULL, NULL,}, + {NULL, NULL, NULL,}, +}; +static const twistshift_ylimdg_kern_list ser_twistshift_ylimdg_list_2v_yShp2[] = { + {NULL, NULL, NULL,}, + {NULL, NULL, NULL,}, +}; +static const twistshift_fullcell_kern_list ser_twistshift_fullcell_list_2v_yShp2[] = { + {NULL, NULL, NULL,}, + {NULL, NULL, NULL,}, +}; + + +struct gkyl_twistshift_dg_kernels { + twistshift_xlimdg_t xlimdg; + twistshift_ylimdg_t ylimdg; + twistshift_fullcell_t fullcell; +}; + +struct ts_shift_dg_eval_ctx { + struct gkyl_array *shift_dg; // DG representation of the shift function. + struct gkyl_basis *shift_b; // Basis for the shift. + struct gkyl_rect_grid *shear_grid; // shear grid along x. + struct gkyl_range *shear_r; // Shear grid range. +}; + +// Primary struct in this updater. +struct gkyl_twistshift_dg { + int bc_dir; // Direction of the BC is applied in. + int shift_dir; // Direction of the shift. + int shear_dir; // Direction the shift varies in (shear). + enum gkyl_edge_loc edge; // Indicates if BC is for lowe/upper edge. + struct gkyl_basis basis; // Basis the shifted field is defined with. + struct gkyl_range local_bcdir_ext_r; // Local range. + struct gkyl_rect_grid grid; // Grid the shifted field is defined in. + evalf_t shift_func; // Function defining the shift. + void *shift_func_ctx; // Context for shift_func. + struct ts_shift_dg_eval_ctx shift_dg_eval_ctx; // Context for DG shift_func. + bool use_gpu; // Whether to apply the BC on the GPU. + + struct gkyl_rect_grid shift_grid; // 1D grid in the direction of the shift. + struct gkyl_range shift_r; // 1D range in the direction of the shift. + + struct gkyl_rect_grid shear_grid; // 1D grid in the direction of the shear. + struct gkyl_range shear_r; // 1D range in the direction of the shear. + + struct gkyl_rect_grid ts_grid; // Grid the shift twistshift takes place in. + struct gkyl_range ts_r; // Range the twistshift takes place in. + int shift_dir_in_ts_grid; // Dimension the shift is in, in the TS grid. + int shear_dir_in_ts_grid; // Dimension the shear is in, in the TS grid. + + int shift_poly_order; // Poly order of the DG representation of the shift. + struct gkyl_basis shift_b; // 1D Basis for the DG shift. + struct gkyl_array *shift_dg; // DG shift. + + int *num_do; // Number of donors at each cell in shear_dir; + int *shift_dir_idx_do; // Indices of donor cells, in the direction of the + // shift, for each cell in the TS grid. + + struct gkyl_twistshift_dg_kernels *kernels; // kernels for sub-cell integrals. + + // Projection object used in constructing the matrices. + struct gkyl_eval_on_nodes *ev_on_nod1d; + // Evaluations of a function at 1D nodes. + struct gkyl_array *func_nod1d; + + struct gkyl_nmat *scimat; // Subcell integral matrices. + struct gkyl_nmat *fmat; // Distribution function matrices. + struct gkyl_nmat *mm_contr; // Contribution resulting from a mat-mat mult. + + long *num_numcol_fidx_do; // 1D indexer, from a index identitying the num-numcol + // plane (in the num-numcol-num_basis space), to a + // linear index into the donor distribution function f. + + long *num_numcol_fidx_tar; // 1D indexer, from a index identitying the num-numcol + // plane (in the num-numcol-num_basis space), to a + // linear index into the target distribution function f. + + int *num_do_cum; // Cumulative number of donors up to a give cell in shear_dir; + struct gkyl_range permutted_ghost_r; // Ghost range to populate in the target + // field, with some dimensions permutted. + struct gkyl_range ghost_r; // Ghost range this BC fills. +}; + +#ifdef GKYL_HAVE_CUDA +/** + * Apply the twist-shift on the NVIDIA GPU. It assumes that periodicity along bc_dir has been + * applied to the donor field. Can be used in-place. + * + * @param up Twist-shift BC updater object. + * @param fdo Donor field. + * @param ftar Target field. + */ +void gkyl_twistshift_dg_advance_cu(struct gkyl_twistshift_dg *up, struct gkyl_array *fdo, struct gkyl_array *ftar); +#endif diff --git a/gyrokinetic/zero/twistshift_dg.c b/gyrokinetic/zero/twistshift_dg.c new file mode 100644 index 0000000000..fc16383692 --- /dev/null +++ b/gyrokinetic/zero/twistshift_dg.c @@ -0,0 +1,2040 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Notes: +// a) Hard-coded parameters: +// - wrap_to_range: eps. +// - find_donors: delta_frac, num_test_pt. +// - find_intersect: tol, max_iter, num_steps. +// - calc_mats: shift_dir_idx_tar. +// - tol_xi: Minimum allowed spacing between the lower and +// upper xi (logical x) limits of subcell integral. +// b) Unlike the procedures described in M. Francisquez, et al. CPC 298 +// (2024) 109109, all subcell integrals are now done with variable y limits. +// This is possible once we realize that figure 4 is not drawn accurately; +// the blue lines should be separated by Delta y at all points. +// c) This updater only works on 5D distributions. Likely only minor changes +// are needed to make it work in other dimensions. +// d) 99% of the code is written to support a BC, a shift and shear in any +// direction. Maybe the only thing that needs to change is the permutted +// range and its use. +// +// List of functions used in computing sub-cell integrals (scimat). +// - ts_grid_cell_boundary_in_dir: cell boundary coordinate in given dir. +// - ts_grid_cell_boundaries: get all cell boundary coords. +// - ts_p2l: physical to logical transform. +// - ts_interval_dx_and_xc: compute length and center of an interval. +// - ts_grid_length_in_dir: length of the grid in given dir. +// - ts_wrap_to_range: wrap a number to a range assuming periodicity. +// - ts_shift_dir_idx_do_linidx: linear index to first donor of a given target +// cell in shift_dir_idx_do. +// - ts_check_shifted_test_point: evaluate a shifted point's cell as a +// potential donor cell. +// - ts_find_donors: find and record the donor cells for each target. +// - ts_root_find: Finds the root of a given function. +// - ts_shifted_coord_loss_func: Loss function used to find where yTar-S +// intersects yDo. +// - ts_sign: return the sign of a double. +// - ts_ts_donor_target_offset: offset between donor and target cells. +// - ts_find_intersect: finds the intersection of yTar-S and yDo. +// - ts_comp_to_phys: transform a computational to a physical coord. +// - ts_nod2mod_proj_1d: evaluate a 1D function at nodes and do a n2m transform +// to get the coefficients of the DG representation. +// - ts_integral_xlimdg: subcell integral with variable x limits. +// - ts_integral_ylimdg: subcell integral with variable y limits. +// - ts_integral_fullcelllimdg: integral over the whole cell. +// - ts_one: return 1 (for projections). +// - ts_minus_one: return -1 (for projections). +// - ts_shift_coord_shifted_log: coordinate in shift_dir shifted and transformed +// to logical space. +// - ts_subcellint_sNi_sNii: subcell integral sNi or sNii. +// - ts_subcellint_si_sii: subcell integral si or sii. +// - ts_subcellint_siii_siv: subcell integral siii or siv. +// - ts_subcellint_sv_svi: subcell integral sv or svi. +// - ts_subcellint_svii_sviii: subcell integral svii or sviii. +// - ts_subcellint_six_sx: subcell integral six or sx. +// - ts_subcellint_sxi_sxii: subcell integral sxi or sxii. +// - ts_subcellint_sxiii_sxiv: subcell integral sxiii or sxiv. +// - ts_subcellint_sxv_sxvi: subcell integral sxv or sxvi. +// - ts_calc_mats: create scimat with the result of the subcell integrals. +// +// Two additional helper functions: +// - ts_calc_num_numcol_fidx_do: index map to populate fmat with donors. +// - ts_calc_num_numcol_fidx_tar: index map to get mat-mat mult results. + +// Option to use the user-provided function describing the shift +// or a DG representation of it: +// = 0 DG representation (default and preferred). +// = 1 user-provided shift function. +// Note: the kernels that ultimately perform the integrals +// always use the DG representation. +#define shift_func_op 0 + +// Minimum allowed spacing between the lower and +// upper xi (logical x) limits of subcell integral. +#define tol_xi 1.0e-15 + +// Indices in 4-element cell boundary array. +#define cellb_lo(dir) (2*dir) +#define cellb_up(dir) (2*dir+1) + +double +ts_grid_cell_boundary_in_dir(struct gkyl_rect_grid *grid, const int *idx, enum gkyl_edge_loc edge, int dir) +{ + // Get the coordinate of the cell boundary in specified direction. + double xc[grid->ndim]; + gkyl_rect_grid_cell_center(grid, idx, xc); + return edge == GKYL_LOWER_EDGE? xc[dir]-0.5*grid->dx[dir] : xc[dir]+0.5*grid->dx[dir]; +} + +void +ts_grid_cell_boundaries(struct gkyl_rect_grid *grid, const int *idx, double *cell_bounds) +{ + // Get the cell boundaries in every dimension. The array cell_bounds + // must be a 2*grid->ndim array. + for (int d=0; dndim; d++) { + cell_bounds[d*2] = ts_grid_cell_boundary_in_dir(grid, idx, GKYL_LOWER_EDGE, d); + cell_bounds[d*2+1] = ts_grid_cell_boundary_in_dir(grid, idx, GKYL_UPPER_EDGE, d); + } +} + +static inline double +ts_p2l(double coord, double cell_center, double dx) +{ + // Transform a physical coordinate (coord) to the [-1,1] logical + // space in a cell centered at cell_center and with length dx. + return 2.0*(coord - cell_center)/dx; +} + +// Evaluation of the shift through the DG representation. +static inline void +ts_shift_dg_eval(double t, const double *coord, double *fout, void *ctx) +{ + struct ts_shift_dg_eval_ctx *tsectx = ctx; + + int cell_idx[GKYL_MAX_DIM]; + gkyl_rect_grid_coord_idx(tsectx->shear_grid, coord, cell_idx); + // Ensure that we do not go outside of the range + // (it does sometimes if x=x_max,x_min). + cell_idx[0] = fmin(cell_idx[0], tsectx->shear_r->upper[0]); + cell_idx[0] = fmax(cell_idx[0], tsectx->shear_r->lower[0]); + + double xc[GKYL_MAX_DIM]; + gkyl_rect_grid_cell_center(tsectx->shear_grid, cell_idx, xc); + + long shift_loc = gkyl_range_idx(tsectx->shear_r, cell_idx); + double *shift_c = (double *) gkyl_array_fetch(tsectx->shift_dg, shift_loc); + double xp = ts_p2l(coord[0], xc[0], tsectx->shear_grid->dx[0]); + + fout[0] = tsectx->shift_b->eval_expand(&(double) {xp}, shift_c); +} + +void +ts_interval_dx_and_xc(const double *interval, double *dx, double *xc) +{ + // Compute the lenth (dx) and center (xc) of [interval[0], interval[1]]. + double lo = interval[0], up = interval[1]; + dx[0] = up - lo; + xc[0] = 0.5*(up + lo); +} + +static inline double +ts_grid_length_in_dir(struct gkyl_rect_grid *grid, int dir) +{ + return grid->upper[dir] - grid->lower[dir]; +} + +double +ts_wrap_to_range(double val, double lower, double upper, bool pick_upper) +{ + // Wrap a number to range [lower,upper]. If pickUpper=true, output upper when + // val is a multiple of upper. Otherwise multiples of upper wrap to lower. + double L = upper - lower; + double disp = fmod(val - lower, L); + double vwrapped = lower + fmod(L + disp, L); + double eps = 1.e-12; + if ( (lower-eps < vwrapped && vwrapped < lower + eps) || + (upper-eps < vwrapped && vwrapped < upper + eps) ) { + if (pick_upper) + return upper; + else + return lower; + } + else + return vwrapped; +} + +long +ts_shift_dir_idx_do_linidx(const int *num_do, int shear_dir_idx, int shift_dir_idx, + int shift_dir_num_cells, int shear_r_lower) +{ + // Return the linear index to the first donor for the idx=(i,j) target cell, + // in the shift_dir_idx_do array. We assume shift_dir_idx_do (whose dimensions + // are Nx,Ny,num_do(i)) is in row-major order, and that it has num_do donors + // at each cell in the shear_dir_in_ts_grid direction. + long linc = 0; + // Count the number of donors in cells with an idx in the shear dir lower + // than this one. NOTE: the -1 here is because the idx is often 1-index + // (since ghost cells are the 0th index) but num_do is only defined on the + // local range. + for (int i=0; ishear_dir_in_ts_grid]}; + int shift_idx[] = {idx[up->shift_dir_in_ts_grid]}; + + int *shift_dir_idx_do_buff_ptr = (int *) gkyl_mem_buff_data(shift_dir_idx_do_buff); + + // Evaluate the shift at this test point. + double test_pt_in_shear_dir = test_pt[up->shear_dir_in_ts_grid]; + double xc_in_shear_dir = xc[up->shear_dir_in_ts_grid]; + double dx_in_shear_dir = dx[up->shear_dir_in_ts_grid]; + double shift_at_pt = up->shift_b.eval_expand( + &(double) {ts_p2l(test_pt_in_shear_dir, xc_in_shear_dir, dx_in_shear_dir)}, shift_c); + + // Find the index of the cell that owns the shifted point. + double shifted_test_pt[] = { ts_wrap_to_range(test_pt[up->shift_dir_in_ts_grid] - shift_at_pt, + up->ts_grid.lower[up->shift_dir_in_ts_grid], up->ts_grid.upper[up->shift_dir_in_ts_grid], + false) }; // Shifted test point. + int shift_dir_idx_test_pt[1]; + gkyl_rect_grid_find_cell(&up->shift_grid, shifted_test_pt, (bool[]) {pick_lower}, (int[]) {-1}, shift_dir_idx_test_pt); + + // Get the linear index to the list of donors for this target. + long linidx = ts_shift_dir_idx_do_linidx(up->num_do, + shear_idx[0], shift_idx[0], up->ts_grid.cells[up->shift_dir_in_ts_grid], up->shear_r.lower[0]); + // If this donor is not in our list of donors, include it. + bool donor_not_found = true; + for (int k=0; k 0) { + // Insert one more int. Only if num_do_curr>0 because we already + // allocated space for the first donor. + size_t new_buff_sz = gkyl_mem_buff_size(shift_dir_idx_do_buff) + sizeof(int); + shift_dir_idx_do_buff = gkyl_mem_buff_resize(shift_dir_idx_do_buff, new_buff_sz); + } + + // Get the pointer again in case it changed. + shift_dir_idx_do_buff_ptr = (int *) gkyl_mem_buff_data(shift_dir_idx_do_buff); + shift_dir_idx_do_buff_ptr[linidx+num_do_curr[0]] = shift_dir_idx_test_pt[0]; + + num_do_curr[0] += 1; + } +} + +void +ts_find_donors(struct gkyl_twistshift_dg *up) +{ + // Find the donor cells for each target cell in the TS grid. + + double delta_frac = 1.e-4; // Distance away from the boundary, as fraction of cell length. + // Must be larger than the eps=1e-6 tolerance in is_in_cell (used by + // gkyl_rect_grid_find_cell) so that shifted test points are never + // ambiguously on a cell boundary. + int num_test_pt[2] = {10, 10}; // Number of test points taken along each side of the cell. + + double step_sz[2] = {0.0}; // Size of the step between test points. + double delta[2] = {0.0}; // Space between cell boundary and test points. + for (int d=0; d<2; d++) { + delta[d] = delta_frac*up->ts_grid.dx[d]; + step_sz[d] = (up->ts_grid.dx[d] - 2.0*delta[d])/(num_test_pt[d]-1); + } + + // Number of donors at each cell of the shear direction. + up->num_do = (int*) gkyl_malloc(up->shear_r.volume * sizeof(int)); + for (int i=0; ishear_r.volume; i++) + up->num_do[i] = -1; + + // Temporary buffer to store donors at (resized below). + size_t curr_buff_sz = up->ts_r.volume * sizeof(int); + gkyl_mem_buff shift_dir_idx_do_buff = gkyl_mem_buff_new(curr_buff_sz); + + int idx[] = {up->shear_r.lower[0]}; + long linidx = gkyl_range_idx(&up->shear_r, idx); + struct gkyl_range_iter iter; + gkyl_range_iter_init(&iter, &up->ts_r); + while (gkyl_range_iter_next(&iter)) { + + // Get the cell boundaries and cell center. + double cell_b[4] = {0.0}; // Cell boundaries, x lo and up, y lo and up; + double xc[2] = {0.0}; // Cell center. + ts_grid_cell_boundaries(&up->ts_grid, iter.idx, cell_b); + gkyl_rect_grid_cell_center(&up->ts_grid, iter.idx, xc); + + int shear_idx[] = {iter.idx[up->shear_dir_in_ts_grid]}; + int shift_idx[] = {iter.idx[up->shift_dir_in_ts_grid]}; + long shift_loc = gkyl_range_idx(&up->shear_r, shear_idx); + double *shift_c = (double *) gkyl_array_fetch(up->shift_dg, shift_loc); + + int num_do_curr = 0; + + for (int dC=0; dC<2; dC++) { // dC=0: x=const, dC=1: y=const (boundaries). + for (int xS=0; xS<2; xS++) { // xS=0: lower, xS=1 upper (boundary). + + double test_pt[2] = {0.0}; // Test point to shift. + for (int d=0; d<2; d++) + test_pt[d] = cell_b[2*d]+delta[d]; + + // Search first shifted point. Use pick_lower=false in find_cell unless + // searching for points along a x=const line near the upper y-boundary. + bool pick_lower = dC==0 && xS==1; + test_pt[dC] += xS*(up->ts_grid.dx[dC]-2.0*delta[dC]); + + // Shift the test point, find the cell that contains it, and if we + // haven't included it yet, add it to our list of donors. + ts_check_shifted_test_point(up, test_pt, xc, up->ts_grid.dx, + shift_c, iter.idx, pick_lower, &num_do_curr, shift_dir_idx_do_buff); + + // Search for other shifted points along this line. + int step_dim = (dC+1) % 2; + for (int sI=1; sIts_grid.dx, + shift_c, iter.idx, pick_lower, &num_do_curr, shift_dir_idx_do_buff); + } + } + } + + up->num_do[shear_idx[0]-up->shear_r.lower[0]] = num_do_curr; + } + + // Copy the donor list to the persistent object and release the buffer. + size_t buff_sz = gkyl_mem_buff_size(shift_dir_idx_do_buff); + up->shift_dir_idx_do = (int *) gkyl_malloc(buff_sz); + int *shift_dir_idx_do_buff_ptr = (int *) gkyl_mem_buff_data(shift_dir_idx_do_buff); + memcpy(up->shift_dir_idx_do, shift_dir_idx_do_buff_ptr, buff_sz); + gkyl_mem_buff_release(shift_dir_idx_do_buff); +} + +struct gkyl_qr_res +ts_root_find(double (*func)(double,void*), void *ctx, const double *lims, int max_iter, double tol) +{ + // Use a Ridder's root finder to find the root of func in the interval + // [lims[0],lims[1]] down to a tolerance 'tol'. Return the interval limit + // if the function is smaller than the tolerance there. Return nil if the + // function does not change sign in the interval (interval doesn't contain the root). + double funcLo = func(lims[0], ctx), funcUp = func(lims[1], ctx); +// if (fabs(funcLo) < tol) +// return (struct gkyl_qr_res) {.res=lims[0], .status=0, .nevals=2}; +// else if (fabs(funcUp) < tol) +// return (struct gkyl_qr_res) {.res=lims[1], .status=0, .nevals=2}; +// else { +// if (funcLo*funcUp < 0) +// return gkyl_ridders(func, ctx, lims[0], lims[1], funcLo, funcUp, max_iter, tol); +// else +// return (struct gkyl_qr_res) {.status=1, .nevals=2}; +// } + if (fabs(funcLo) > tol && fabs(funcUp) > tol) { + if (funcLo*funcUp < 0) + return gkyl_ridders(func, ctx, lims[0], lims[1], funcLo, funcUp, max_iter, tol); + else + return (struct gkyl_qr_res) {.status=1, .nevals=2}; + } + else if (fabs(funcLo) < tol && fabs(funcUp) < tol) + return (struct gkyl_qr_res) {.status=1, .nevals=2}; + else if (fabs(funcLo) < tol) + return (struct gkyl_qr_res) {.res=lims[0], .status=0, .nevals=2}; + else if (fabs(funcUp) < tol) + return (struct gkyl_qr_res) {.res=lims[1], .status=0, .nevals=2}; + return (struct gkyl_qr_res) {.status=1, .nevals=2}; +} + +struct ts_shifted_coord_loss_func_ctx { + double shiftCoordTar; // Target coordinate in shift_dir. + double shiftCoordDo; // Donor coordinate in shift_dir. + double shiftDirL; // Length of the domain in shift_dir. + int periodicCopyIdx; // Used to search a periodic copy of the domain (signed). + evalf_t shift_func; // Function defining the shift. + void *shift_func_ctx; // Context for shift_func. +}; + +double ts_shifted_coord_loss_func(double shearCoord, void *ctx) +{ + // Loss function used to find the shear coord. + struct ts_shifted_coord_loss_func_ctx *tsctx = ctx; + + double shift; + tsctx->shift_func(0.0, (double[]){shearCoord}, &shift, tsctx->shift_func_ctx); + + return tsctx->shiftCoordTar - shift + - (tsctx->shiftCoordDo - tsctx->periodicCopyIdx * tsctx->shiftDirL); +} + +int static inline +ts_sign(double a) +{ + if (a < 0.0) + return -1; + else if (a > 0.0) + return 1; + else + return 0; +} + +double +ts_donor_target_offset(struct gkyl_twistshift_dg *up, const double *xc_do, const double *xc_tar) { + // y-offset between the donor and the target cell (yDo-yTar), in the direction of the shift. + // xc_do: cell center coordinates of donor cell. + // xc_tar: cell center coordinates of target cell. + int shift_dir = up->shift_dir_in_ts_grid; + double x_eval = xc_do[up->shear_dir]; + double shift; + up->shift_func(0.0, (double[]){x_eval}, &shift, up->shift_func_ctx); + + double shift_dir_L = up->ts_grid.upper[up->shift_dir] - up->ts_grid.lower[up->shift_dir]; + + // The idea here is that we keep shifting the donor cell center until it is in a + // periodic copy of our domain which overlaps with the shifted target cell center. + double xs_shifted_do = xc_do[shift_dir]; + double xs_shifted_tar = xc_tar[shift_dir] - shift; + // Step toward the target. + int shift_sign = xs_shifted_tar < xs_shifted_do? 1 : -1; + bool keep_shifting = true; + while (keep_shifting) { + double xs_shifted_dolo = xs_shifted_do - shift_dir_L/2.0; + double xs_shifted_doup = xs_shifted_do + shift_dir_L/2.0; + if (xs_shifted_dolo <= xs_shifted_tar && xs_shifted_tar <= xs_shifted_doup) { + keep_shifting = false; + break; + } + else + xs_shifted_do = xs_shifted_do - shift_sign*shift_dir_L; + } + return xc_tar[shift_dir] - xs_shifted_do; +} + +struct gkyl_qr_res +ts_find_intersect(struct gkyl_twistshift_dg *up, double shiftCoordTar, double shiftCoordDo, + const double *shearDirBounds, const double *shiftDirLimits, int nP_primary) +{ + // Given a y-coordinate of the target cell (yTar), and a y-coordinate + // of the donor cell (yDo), find the x-coordinate where yTar-yShift(x)=yDo + // for the periodic copy identified by nP_primary, i.e. the root of + // yTar - yShift(x) - (yDo - nP_primary*Ly) = 0 in [shearDirBounds[0], shearDirBounds[1]]. + // Returns status=1 (not found) if no root exists for that specific nP_primary. + // + // nP_primary must be the same for all 4 inter_pts of a donor-target pair: all four + // intersection conditions describe corners of a single physical overlap diamond and + // therefore belong to the same periodic copy of the domain. + double tol = 1.e-13; + int max_iter = 100; + + double shiftDirL = shiftDirLimits[1] - shiftDirLimits[0]; + + struct ts_shifted_coord_loss_func_ctx func_ctx = { + .shiftCoordTar = shiftCoordTar, + .shiftCoordDo = shiftCoordDo, + .shiftDirL = shiftDirL, + .periodicCopyIdx = nP_primary, + .shift_func = up->shift_func, + .shift_func_ctx = up->shift_func_ctx, + }; + return ts_root_find(ts_shifted_coord_loss_func, &func_ctx, shearDirBounds, max_iter, tol); +} + +struct ts_val_found { + bool status; // =0 not found, =1 found. + double value; // value found. +}; + +static inline void +ts_comp_to_phys(int ndim, const double *eta, + const double * GKYL_RESTRICT dx, const double * GKYL_RESTRICT xc, + double* GKYL_RESTRICT xout) +{ + for (int d=0; dshift_b.num_basis; ++i) { + ts_comp_to_phys(1, gkyl_eval_on_nodes_fetch_node(up->ev_on_nod1d, i), + dx, xc, xmu); + func(0.0, xmu, (double *)gkyl_array_fetch(up->func_nod1d,i), func_ctx); + } + gkyl_eval_on_nodes_nod2mod(up->ev_on_nod1d, up->func_nod1d, out); +} + +void +ts_integral_xlimdg(struct gkyl_twistshift_dg *up, double sFac, const double *xLimLo, + const double *xLimUp, double yLimLo, double yLimUp, double dyDo, double yOff, + const double *ySh, struct gkyl_mat *mat_do) { + // Populate a matrix (mat_do) with a sub-cell integral that has variably x limits + // represented by a DG polynomial, and a y-integral that goes from yLimLo to yLimUp. + // up: BC updater. + // sFac: +/-1 factor to add or subtract this subcell integral. + // xLimLo: DG representation of the lower x-limit. + // xLimUp: DG representation of the upper x-limit. + // yLimLo: lower y-limit. + // yLimUp: upper y-limit. + // dyDo: Cell length along y. + // yOff: Offset along y. + // ySh: DG representation of the y shift. + // mat_do: donor matrix. + up->kernels->xlimdg(sFac, xLimLo, xLimUp, yLimLo, yLimUp, dyDo, yOff, ySh, mat_do); +} + +void +ts_integral_ylimdg(struct gkyl_twistshift_dg *up, double sFac, double xLimLo, double xLimUp, + const double *yLimLo, const double *yLimUp, double dyDo, double yOff, + const double *ySh, struct gkyl_mat *mat_do) { + // Populate a matrix (mat_do) with a sub-cell integral that has variable y limits + // represented by a DG polynomial, and a x-integral that goes from xLimLo to xLimUp. + // up: BC updater. + // sFac: +/-1 factor to add or subtract this subcell integral. + // xLimLo: lower x-limit. + // xLimUp: upper x-limit. + // yLimLo: DG representation of the lower y-limit. + // yLimUp: DG representation of the upper y-limit. + // dyDo: Cell length along y. + // yOff: Offset along y. + // ySh: DG representation of the y shift. + // mat_do: donor matrix. + up->kernels->ylimdg(sFac, xLimLo, xLimUp, yLimLo, yLimUp, dyDo, yOff, ySh, mat_do); +} + +void +ts_integral_fullcelllimdg(struct gkyl_twistshift_dg *up, double dyDo, double yOff, + const double *ySh, struct gkyl_mat *mat_do) { + // Populate a matrix (mat_do) with the full-cell integral. + // up: BC updater. + // sFac: +/-1 factor to add or subtract this subcell integral. + // dyDo: Cell length along y. + // yOff: Offset along y. + // ySh: DG representation of the y shift. + // mat_do: donor matrix. + up->kernels->fullcell(dyDo, yOff, ySh, mat_do); +} + +static inline void +ts_one(double t, const double *xn, double *fout, void *ctx) +{ + fout[0] = 1.0; +} + +static inline void +ts_minus_one(double t, const double *xn, double *fout, void *ctx) +{ + fout[0] = -1.0; +} + +struct ts_shift_coord_shifted_log_ctx { + double shift_coord_tar; // Target coordinate in shift_dir. + int shift_sign_fac; + const double *xc_do, *xc_tar; // Cell centers (donor and target). + double *dx; // Cell lengths. + bool pick_upper; + int shear_dir, shift_dir; // Shear and shift directions. + double shift_dir_bounds[2]; // Domain boundaries in shift_dir. + evalf_t shift_func; // Function defining the shift. + void *shift_func_ctx; // Context for shift_func. +}; + +void +ts_shift_coord_shifted_log(double t, const double *xn, double *fout, void *ctx) +{ + // Given a logical space x coordinate (xi) and a (physical) y-coordinate in the target cell, + // compute the shifted y-coordinate in the logical space of the donor cell (eta \in [-1,1]). + // xi: logical space x coordinate. + // yTar: physical y-coordinate in target cell. + // pmSh: factor multiplying the y-shift (+/- 1). + // xcDo: cell center coordinates of donor cell. + // xcTar: cell center coordinates of target cell. + // dx: cell lengths. + // pickUpper: boolean indicating if wrapping function should return upper/lower boundary. + + double xi = xn[0]; + + struct ts_shift_coord_shifted_log_ctx *tsctx = ctx; + double shift_coord_tar = tsctx->shift_coord_tar; + int shift_sign_fac = tsctx->shift_sign_fac; + const double *xc_do = tsctx->xc_do, *xc_tar = tsctx->xc_tar; + double *dx = tsctx->dx; + bool pick_upper = tsctx->pick_upper; + int shear_dir = tsctx->shear_dir, shift_dir = tsctx->shift_dir; + double *shift_dir_bounds = tsctx->shift_dir_bounds; + + double shear_coord_phys = xc_tar[shear_dir] + 0.5*dx[shear_dir]*xi; + double shift; + tsctx->shift_func(0.0, (double[]){shear_coord_phys}, &shift, tsctx->shift_func_ctx); + + double shift_coord_shifted = shift_coord_tar - shift_sign_fac * shift; + shift_coord_shifted = ts_wrap_to_range(shift_coord_shifted, shift_dir_bounds[0], shift_dir_bounds[1], pick_upper); + + fout[0] = ts_p2l(shift_coord_shifted, xc_do[shift_dir], dx[shift_dir]); +} + +void +ts_subcellint_sNi_sNii(struct gkyl_twistshift_dg *up, struct ts_val_found *inter_pts, + const double *xc_do, const double *xc_tar, const double *cellb_do, const double *cellb_tar, + bool is_upper_shift_dir_cell, const double *shift_c, struct gkyl_mat *mat_do) +{ + // Perform sNi or sNii subcell integrals. + // inter_pts: intersections y_{j_tar-/+1/2}-yShift and y_{j_do-/+1/2} (lower/upper y-boundaries of donor cell). + // xc_do: cell center of donor cell. + // xc_tar: cell center of target cell. + // cellb_do: boundaries of target cell. + // cellb_tar: boundaries of target cell. + // is_upper_shift_dir_cell: is the donor the upper cell in the shift dir? + // shift_c: DG coefficients of the shift. + // mat_do: current donor matrix. + + bool is_sNi = inter_pts[2].value < inter_pts[0].value; + + struct ts_shift_coord_shifted_log_ctx eta_lims_ctx = { + .shift_sign_fac = 1, + .xc_do = xc_do, + .xc_tar = xc_tar, + .dx = up->ts_grid.dx, + .pick_upper = is_upper_shift_dir_cell, + .shear_dir = up->shear_dir_in_ts_grid, + .shift_dir = up->shift_dir_in_ts_grid, + .shift_dir_bounds = {up->ts_grid.lower[up->shift_dir_in_ts_grid], + up->ts_grid.upper[up->shift_dir_in_ts_grid]}, + .shift_func = up->shift_func, + .shift_func_ctx = up->shift_func_ctx, + }; + + double xi_b[2]; // Limits of xi integral. + evalf_t eta_lims[2]; // Table of functions definting the limits of eta integral. + double etalo_xi[up->shift_b.num_basis], etaup_xi[up->shift_b.num_basis]; + + // Offset between cell centers in direction of the shift. + double xs_off = ts_donor_target_offset(up, xc_do, xc_tar); + + if (is_sNi) { + // sNi + // 1) Add the contribution of the left portion. + xi_b[0] = ts_p2l(inter_pts[2].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + xi_b[1] = ts_p2l(inter_pts[3].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + + eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_up(up->shift_dir_in_ts_grid)]; + eta_lims[0] = ts_minus_one; + eta_lims[1] = ts_shift_coord_shifted_log; + + ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); + ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); + + if (fabs(xi_b[1] - xi_b[0]) > tol_xi) + up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, + up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); + + // 2) Add the contribution of the right portion. + xi_b[0] = ts_p2l(inter_pts[0].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + xi_b[1] = ts_p2l(inter_pts[1].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + + eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_lo(up->shift_dir_in_ts_grid)]; + eta_lims[0] = ts_shift_coord_shifted_log; + eta_lims[1] = ts_one; + + ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); + ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); + + if (fabs(xi_b[1] - xi_b[0]) > tol_xi) + up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, + up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); + } + else { + // sNii + // 1) Add the contribution of the left portion. + xi_b[0] = ts_p2l(inter_pts[1].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + xi_b[1] = ts_p2l(inter_pts[0].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + + eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_lo(up->shift_dir_in_ts_grid)]; + eta_lims[0] = ts_shift_coord_shifted_log; + eta_lims[1] = ts_one; + + ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); + ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); + + if (fabs(xi_b[1] - xi_b[0]) > tol_xi) + up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, + up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); + + // 2) Add the contribution of the right portion. + xi_b[0] = ts_p2l(inter_pts[3].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + xi_b[1] = ts_p2l(inter_pts[2].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + + eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_up(up->shift_dir_in_ts_grid)]; + eta_lims[0] = ts_minus_one; + eta_lims[1] = ts_shift_coord_shifted_log; + + ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); + ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); + + if (fabs(xi_b[1] - xi_b[0]) > tol_xi) + up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, + up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); + } +} + +void +ts_subcellint_si_sii(struct gkyl_twistshift_dg *up, struct ts_val_found *inter_pts, + const double *xc_do, const double *xc_tar, const double *cellb_do, const double *cellb_tar, + bool is_upper_shift_dir_cell, const double *shift_c, struct gkyl_mat *mat_do) +{ + // Perform subcell integral si or sii, using fixed x-limits and variable y limits. + // inter_pts: intersections y_{j_tar-/+1/2}-yShift and y_{j_do-/+1/2} (lower/upper y-boundaries of donor cell). + // xc_do: cell center of donor cell. + // xc_tar: cell center of target cell. + // cellb_do: boundaries of target cell. + // cellb_tar: boundaries of target cell. + // is_upper_shift_dir_cell: is the donor the upper cell in the shift dir? + // shift_c: DG coefficients of the shift. + // mat_do: current donor matrix. + + double x_up = cellb_tar[cellb_up(up->shear_dir_in_ts_grid)]; + double x_lo = cellb_tar[cellb_lo(up->shear_dir_in_ts_grid)]; + double shift_lo, shift_up; + + up->shift_func(0.0, (double[]){x_lo}, &shift_lo, up->shift_func_ctx); + up->shift_func(0.0, (double[]){x_up}, &shift_up, up->shift_func_ctx); + + bool is_si = -shift_lo < -shift_up; + + double xi_b[2]; // Limits of xi integral. + if (is_si) { + // si integral. + xi_b[0] = -1.0; + xi_b[1] = ts_p2l(inter_pts[1].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]);; + } + else { + // sii integral. + xi_b[0] = ts_p2l(inter_pts[1].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]);; + xi_b[1] = 1.0; + } + + evalf_t eta_lims[2]; // Table of functions definting the limits of eta integral. + eta_lims[0] = ts_shift_coord_shifted_log; + eta_lims[1] = ts_one; + + struct ts_shift_coord_shifted_log_ctx eta_lims_ctx = { + .shift_coord_tar = cellb_tar[cellb_lo(up->shift_dir_in_ts_grid)], + .shift_sign_fac = 1, + .xc_do = xc_do, + .xc_tar = xc_tar, + .dx = up->ts_grid.dx, + .pick_upper = is_upper_shift_dir_cell, + .shear_dir = up->shear_dir_in_ts_grid, + .shift_dir = up->shift_dir_in_ts_grid, + .shift_dir_bounds = {up->ts_grid.lower[up->shift_dir_in_ts_grid], + up->ts_grid.upper[up->shift_dir_in_ts_grid]}, + .shift_func = up->shift_func, + .shift_func_ctx = up->shift_func_ctx, + }; + + double etalo_xi[up->shift_b.num_basis], etaup_xi[up->shift_b.num_basis]; + ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); + ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); + // Offset between cell centers in direction of the shift. + double xs_off = ts_donor_target_offset(up, xc_do, xc_tar); + + if (fabs(xi_b[1] - xi_b[0]) > tol_xi) + up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, + up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); +} + +void +ts_subcellint_siii_siv(struct gkyl_twistshift_dg *up, struct ts_val_found *inter_pts, + const double *xc_do, const double *xc_tar, const double *cellb_do, const double *cellb_tar, + bool is_upper_shift_dir_cell, const double *shift_c, struct gkyl_mat *mat_do) +{ + // Perform subcell integral siii or siv, using fixed x-limits and variable y limits. + // inter_pts: intersections y_{j_tar-/+1/2}-yShift and y_{j_do-/+1/2} (lower/upper y-boundaries of donor cell). + // xc_do: cell center of donor cell. + // xc_tar: cell center of target cell. + // cellb_do: boundaries of target cell. + // cellb_tar: boundaries of target cell. + // is_upper_shift_dir_cell: is the donor the upper cell in the shift dir? + // shift_c: DG coefficients of the shift. + // mat_do: current donor matrix. + + double x_lo = cellb_tar[cellb_lo(up->shear_dir_in_ts_grid)]; + double x_up = cellb_tar[cellb_up(up->shear_dir_in_ts_grid)]; + double shift_lo, shift_up; + + up->shift_func(0.0, (double[]){x_lo}, &shift_lo, up->shift_func_ctx); + up->shift_func(0.0, (double[]){x_up}, &shift_up, up->shift_func_ctx); + + bool is_siii = -shift_lo > -shift_up; + + double xi_b[2]; // Limits of xi integral. + if (is_siii) { + // siii integral. + xi_b[0] = -1.0; + xi_b[1] = ts_p2l(inter_pts[2].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]);; + } + else { + // siv integral. + xi_b[0] = ts_p2l(inter_pts[2].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]);; + xi_b[1] = 1.0; + } + + evalf_t eta_lims[2]; // Table of functions definting the limits of eta integral. + eta_lims[0] = ts_minus_one; + eta_lims[1] = ts_shift_coord_shifted_log; + + struct ts_shift_coord_shifted_log_ctx eta_lims_ctx = { + .shift_coord_tar = cellb_tar[cellb_up(up->shift_dir_in_ts_grid)], + .shift_sign_fac = 1, + .xc_do = xc_do, + .xc_tar = xc_tar, + .dx = up->ts_grid.dx, + .pick_upper = is_upper_shift_dir_cell, + .shear_dir = up->shear_dir_in_ts_grid, + .shift_dir = up->shift_dir_in_ts_grid, + .shift_dir_bounds = {up->ts_grid.lower[up->shift_dir_in_ts_grid], + up->ts_grid.upper[up->shift_dir_in_ts_grid]}, + .shift_func = up->shift_func, + .shift_func_ctx = up->shift_func_ctx, + }; + + double etalo_xi[up->shift_b.num_basis], etaup_xi[up->shift_b.num_basis]; + ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); + ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); + // Offset between cell centers in direction of the shift. + double xs_off = ts_donor_target_offset(up, xc_do, xc_tar); + + if (fabs(xi_b[1] - xi_b[0]) > tol_xi) + up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, + up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); +} + +void +ts_subcellint_sv_svi(struct gkyl_twistshift_dg *up, struct ts_val_found *inter_pts, + const double *xc_do, const double *xc_tar, const double *cellb_do, const double *cellb_tar, + bool is_upper_shift_dir_cell, const double *shift_c, struct gkyl_mat *mat_do) +{ + // Perform sv or svi subcell integrals. + // inter_pts: intersections y_{j_tar-/+1/2}-yShift and y_{j_do-/+1/2} (lower/upper y-boundaries of donor cell). + // xc_do: cell center of donor cell. + // xc_tar: cell center of target cell. + // cellb_do: boundaries of target cell. + // cellb_tar: boundaries of target cell. + // is_upper_shift_dir_cell: is the donor the upper cell in the shift dir? + // shift_c: DG coefficients of the shift. + // mat_do: current donor matrix. + + bool is_sv = inter_pts[3].value < inter_pts[1].value; + + struct ts_shift_coord_shifted_log_ctx eta_lims_ctx = { + .shift_sign_fac = 1, + .xc_do = xc_do, + .xc_tar = xc_tar, + .dx = up->ts_grid.dx, + .pick_upper = is_upper_shift_dir_cell, + .shear_dir = up->shear_dir_in_ts_grid, + .shift_dir = up->shift_dir_in_ts_grid, + .shift_dir_bounds = {up->ts_grid.lower[up->shift_dir_in_ts_grid], + up->ts_grid.upper[up->shift_dir_in_ts_grid]}, + .shift_func = up->shift_func, + .shift_func_ctx = up->shift_func_ctx, + }; + + double xi_b[2]; // Limits of xi integral. + evalf_t eta_lims[2]; // Table of functions definting the limits of eta integral. + double etalo_xi[up->shift_b.num_basis], etaup_xi[up->shift_b.num_basis]; + + // Offset between cell centers in direction of the shift. + double xs_off = ts_donor_target_offset(up, xc_do, xc_tar); + + if (is_sv) { + // sv + // 1) Add the contribution of the left portion. + xi_b[0] = -1.0; + xi_b[1] = ts_p2l(inter_pts[3].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + + eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_up(up->shift_dir_in_ts_grid)]; + eta_lims[0] = ts_minus_one; + eta_lims[1] = ts_shift_coord_shifted_log; + + ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); + ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); + + if (fabs(xi_b[1] - xi_b[0]) > tol_xi) + up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, + up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); + + // 2) Add the contribution of the right portion. + xi_b[0] = ts_p2l(inter_pts[0].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + xi_b[1] = ts_p2l(inter_pts[1].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + + eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_lo(up->shift_dir_in_ts_grid)]; + eta_lims[0] = ts_shift_coord_shifted_log; + eta_lims[1] = ts_one; + + ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); + ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); + + if (fabs(xi_b[1] - xi_b[0]) > tol_xi) + up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, + up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); + } + else { + // svi + // 1) Add the contribution of the left portion. + xi_b[0] = ts_p2l(inter_pts[1].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + xi_b[1] = ts_p2l(inter_pts[0].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + + eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_lo(up->shift_dir_in_ts_grid)]; + eta_lims[0] = ts_shift_coord_shifted_log; + eta_lims[1] = ts_one; + + ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); + ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); + + if (fabs(xi_b[1] - xi_b[0]) > tol_xi) + up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, + up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); + + // 2) Add the contribution of the right portion. + xi_b[0] = ts_p2l(inter_pts[3].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + xi_b[1] = 1.0; + + eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_up(up->shift_dir_in_ts_grid)]; + eta_lims[0] = ts_minus_one; + eta_lims[1] = ts_shift_coord_shifted_log; + + ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); + ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); + + if (fabs(xi_b[1] - xi_b[0]) > tol_xi) + up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, + up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); + } +} + +void +ts_subcellint_svii_sviii(struct gkyl_twistshift_dg *up, struct ts_val_found *inter_pts, + const double *xc_do, const double *xc_tar, const double *cellb_do, const double *cellb_tar, + bool is_upper_shift_dir_cell, const double *shift_c, struct gkyl_mat *mat_do) +{ + // Perform svii or sviii subcell integrals. + // inter_pts: intersections y_{j_tar-/+1/2}-yShift and y_{j_do-/+1/2} (lower/upper y-boundaries of donor cell). + // xc_do: cell center of donor cell. + // xc_tar: cell center of target cell. + // cellb_do: boundaries of target cell. + // cellb_tar: boundaries of target cell. + // is_upper_shift_dir_cell: is the donor the upper cell in the shift dir? + // shift_c: DG coefficients of the shift. + // mat_do: current donor matrix. + + bool is_svii = inter_pts[0].value < inter_pts[2].value; + + struct ts_shift_coord_shifted_log_ctx eta_lims_ctx = { + .shift_sign_fac = 1, + .xc_do = xc_do, + .xc_tar = xc_tar, + .dx = up->ts_grid.dx, + .pick_upper = is_upper_shift_dir_cell, + .shear_dir = up->shear_dir_in_ts_grid, + .shift_dir = up->shift_dir_in_ts_grid, + .shift_dir_bounds = {up->ts_grid.lower[up->shift_dir_in_ts_grid], + up->ts_grid.upper[up->shift_dir_in_ts_grid]}, + .shift_func = up->shift_func, + .shift_func_ctx = up->shift_func_ctx, + }; + + double xi_b[2]; // Limits of xi integral. + evalf_t eta_lims[2]; // Table of functions definting the limits of eta integral. + double etalo_xi[up->shift_b.num_basis], etaup_xi[up->shift_b.num_basis]; + + // Offset between cell centers in direction of the shift. + double xs_off = ts_donor_target_offset(up, xc_do, xc_tar); + + if (is_svii) { + // svii + // 1) Add the contribution of the left portion. + xi_b[0] = -1.0; + xi_b[1] = ts_p2l(inter_pts[0].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + + eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_lo(up->shift_dir_in_ts_grid)]; + eta_lims[0] = ts_shift_coord_shifted_log; + eta_lims[1] = ts_one; + + ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); + ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); + + if (fabs(xi_b[1] - xi_b[0]) > tol_xi) + up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, + up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); + + // 2) Add the contribution of the right portion. + xi_b[0] = ts_p2l(inter_pts[3].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + xi_b[1] = ts_p2l(inter_pts[2].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + + eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_up(up->shift_dir_in_ts_grid)]; + eta_lims[0] = ts_minus_one; + eta_lims[1] = ts_shift_coord_shifted_log; + + ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); + ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); + + if (fabs(xi_b[1] - xi_b[0]) > tol_xi) + up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, + up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); + } + else { + // sviii + // 1) Add the contribution of the left portion. + xi_b[0] = ts_p2l(inter_pts[2].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + xi_b[1] = ts_p2l(inter_pts[3].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + + eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_up(up->shift_dir_in_ts_grid)]; + eta_lims[0] = ts_minus_one; + eta_lims[1] = ts_shift_coord_shifted_log; + + ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); + ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); + + if (fabs(xi_b[1] - xi_b[0]) > tol_xi) + up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, + up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); + + // 2) Add the contribution of the right portion. + xi_b[0] = ts_p2l(inter_pts[0].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + xi_b[1] = 1.0; + + eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_lo(up->shift_dir_in_ts_grid)]; + eta_lims[0] = ts_shift_coord_shifted_log; + eta_lims[1] = ts_one; + + ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); + ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); + + if (fabs(xi_b[1] - xi_b[0]) > tol_xi) + up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, + up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); + } +} + +void +ts_subcellint_six_sx(struct gkyl_twistshift_dg *up, struct ts_val_found *inter_pts, + const double *xc_do, const double *xc_tar, const double *cellb_do, const double *cellb_tar, + bool is_upper_shift_dir_cell, const double *shift_c, struct gkyl_mat *mat_do) +{ + // Perform six or sx subcell integrals. + // inter_pts: intersections y_{j_tar-/+1/2}-yShift and y_{j_do-/+1/2} (lower/upper y-boundaries of donor cell). + // xc_do: cell center of donor cell. + // xc_tar: cell center of target cell. + // cellb_do: boundaries of target cell. + // cellb_tar: boundaries of target cell. + // is_upper_shift_dir_cell: is the donor the upper cell in the shift dir? + // shift_c: DG coefficients of the shift. + // mat_do: current donor matrix. + + bool is_six = inter_pts[0].value < inter_pts[1].value; + + // Limits of xi integral. + double xi_b[2]; + if (is_six) { + // six + xi_b[0] = ts_p2l(inter_pts[0].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + xi_b[1] = ts_p2l(inter_pts[1].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + } + else { + // sx + xi_b[0] = ts_p2l(inter_pts[1].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + xi_b[1] = ts_p2l(inter_pts[0].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + } + + struct ts_shift_coord_shifted_log_ctx eta_lims_ctx = { + .shift_coord_tar = cellb_tar[cellb_lo(up->shift_dir_in_ts_grid)], + .shift_sign_fac = 1, + .xc_do = xc_do, + .xc_tar = xc_tar, + .dx = up->ts_grid.dx, + .pick_upper = is_upper_shift_dir_cell, + .shear_dir = up->shear_dir_in_ts_grid, + .shift_dir = up->shift_dir_in_ts_grid, + .shift_dir_bounds = {up->ts_grid.lower[up->shift_dir_in_ts_grid], + up->ts_grid.upper[up->shift_dir_in_ts_grid]}, + .shift_func = up->shift_func, + .shift_func_ctx = up->shift_func_ctx, + }; + + evalf_t eta_lims[2]; // Table of functions definting the limits of eta integral. + eta_lims[0] = ts_shift_coord_shifted_log; + eta_lims[1] = ts_one; + + double etalo_xi[up->shift_b.num_basis], etaup_xi[up->shift_b.num_basis]; + ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); + ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); + // Offset between cell centers in direction of the shift. + double xs_off = ts_donor_target_offset(up, xc_do, xc_tar); + + if (fabs(xi_b[1] - xi_b[0]) > tol_xi) + up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, + up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); +} + +void +ts_subcellint_sxi_sxii(struct gkyl_twistshift_dg *up, struct ts_val_found *inter_pts, + const double *xc_do, const double *xc_tar, const double *cellb_do, const double *cellb_tar, + bool is_upper_shift_dir_cell, const double *shift_c, struct gkyl_mat *mat_do) +{ + // Perform sxi or sxii subcell integrals. + // inter_pts: intersections y_{j_tar-/+1/2}-yShift and y_{j_do-/+1/2} (lower/upper y-boundaries of donor cell). + // xc_do: cell center of donor cell. + // xc_tar: cell center of target cell. + // cellb_do: boundaries of target cell. + // cellb_tar: boundaries of target cell. + // is_upper_shift_dir_cell: is the donor the upper cell in the shift dir? + // shift_c: DG coefficients of the shift. + // mat_do: current donor matrix. + + bool is_sxi = inter_pts[3].value < inter_pts[2].value; + + // Limits of xi integral. + double xi_b[2]; + if (is_sxi) { + // six + xi_b[0] = ts_p2l(inter_pts[3].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + xi_b[1] = ts_p2l(inter_pts[2].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + } + else { + // sx + xi_b[0] = ts_p2l(inter_pts[2].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + xi_b[1] = ts_p2l(inter_pts[3].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + } + + struct ts_shift_coord_shifted_log_ctx eta_lims_ctx = { + .shift_coord_tar = cellb_tar[cellb_up(up->shift_dir_in_ts_grid)], + .shift_sign_fac = 1, + .xc_do = xc_do, + .xc_tar = xc_tar, + .dx = up->ts_grid.dx, + .pick_upper = is_upper_shift_dir_cell, + .shear_dir = up->shear_dir_in_ts_grid, + .shift_dir = up->shift_dir_in_ts_grid, + .shift_dir_bounds = {up->ts_grid.lower[up->shift_dir_in_ts_grid], + up->ts_grid.upper[up->shift_dir_in_ts_grid]}, + .shift_func = up->shift_func, + .shift_func_ctx = up->shift_func_ctx, + }; + + evalf_t eta_lims[2]; // Table of functions definting the limits of eta integral. + eta_lims[0] = ts_minus_one; + eta_lims[1] = ts_shift_coord_shifted_log; + + double etalo_xi[up->shift_b.num_basis], etaup_xi[up->shift_b.num_basis]; + ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); + ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); + // Offset between cell centers in direction of the shift. + double xs_off = ts_donor_target_offset(up, xc_do, xc_tar); + + if (fabs(xi_b[1] - xi_b[0]) > tol_xi) + up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, + up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); +} + +void +ts_subcellint_sxiii_sxiv(struct gkyl_twistshift_dg *up, struct ts_val_found *inter_pts, + const double *xc_do, const double *xc_tar, const double *cellb_do, const double *cellb_tar, + bool is_upper_shift_dir_cell, const double *shift_c, struct gkyl_mat *mat_do) +{ + // Perform sxiii or sxiv subcell integrals. + // inter_pts: intersections y_{j_tar-/+1/2}-yShift and y_{j_do-/+1/2} (lower/upper y-boundaries of donor cell). + // xc_do: cell center of donor cell. + // xc_tar: cell center of target cell. + // cellb_do: boundaries of target cell. + // cellb_tar: boundaries of target cell. + // is_upper_shift_dir_cell: is the donor the upper cell in the shift dir? + // shift_c: DG coefficients of the shift. + // mat_do: current donor matrix. + + double x_lo = cellb_tar[cellb_lo(up->shear_dir_in_ts_grid)]; + double x_up = cellb_tar[cellb_up(up->shear_dir_in_ts_grid)]; + double shift_lo, shift_up; + + up->shift_func(0.0, (double[]){x_lo}, &shift_lo, up->shift_func_ctx); + up->shift_func(0.0, (double[]){x_up}, &shift_up, up->shift_func_ctx); + + bool is_sxiii = -shift_lo < -shift_up; + + struct ts_shift_coord_shifted_log_ctx eta_lims_ctx = { + .shift_sign_fac = 1, + .xc_do = xc_do, + .xc_tar = xc_tar, + .dx = up->ts_grid.dx, + .pick_upper = is_upper_shift_dir_cell, + .shear_dir = up->shear_dir_in_ts_grid, + .shift_dir = up->shift_dir_in_ts_grid, + .shift_dir_bounds = {up->ts_grid.lower[up->shift_dir_in_ts_grid], + up->ts_grid.upper[up->shift_dir_in_ts_grid]}, + .shift_func = up->shift_func, + .shift_func_ctx = up->shift_func_ctx, + }; + + double xi_b[2]; // Limits of xi integral. + evalf_t eta_lims[2]; // Table of functions definting the limits of eta integral. + double etalo_xi[up->shift_b.num_basis], etaup_xi[up->shift_b.num_basis]; + + // Offset between cell centers in direction of the shift. + double xs_off = ts_donor_target_offset(up, xc_do, xc_tar); + + // 1) Add the contribution of the left portion. + xi_b[0] = -1.0; + xi_b[1] = ts_p2l(inter_pts[3].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + + if (is_sxiii) { + eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_up(up->shift_dir_in_ts_grid)]; + eta_lims[0] = ts_minus_one; + eta_lims[1] = ts_shift_coord_shifted_log; + } + else { + eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_lo(up->shift_dir_in_ts_grid)]; + eta_lims[0] = ts_shift_coord_shifted_log; + eta_lims[1] = ts_one; + } + + ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); + ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); + + if (fabs(xi_b[1] - xi_b[0]) > tol_xi) + up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, + up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); + + // 2) Add the contribution of the right portion. + xi_b[0] = ts_p2l(inter_pts[0].value, xc_do[up->shear_dir_in_ts_grid], up->ts_grid.dx[up->shear_dir_in_ts_grid]); + xi_b[1] = 1.0; + + if (is_sxiii) { + eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_lo(up->shift_dir_in_ts_grid)]; + eta_lims[0] = ts_shift_coord_shifted_log; + eta_lims[1] = ts_one; + } + else { + eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_up(up->shift_dir_in_ts_grid)]; + eta_lims[0] = ts_minus_one; + eta_lims[1] = ts_shift_coord_shifted_log; + } + + ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); + ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); + + if (fabs(xi_b[1] - xi_b[0]) > tol_xi) + up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, + up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); +} + +void +ts_subcellint_sxv_sxvi(struct gkyl_twistshift_dg *up, struct ts_val_found *inter_pts, + const double *xc_do, const double *xc_tar, const double *cellb_do, const double *cellb_tar, + bool is_upper_shift_dir_cell, const double *shift_c, struct gkyl_mat *mat_do) +{ + // Perform subcell integral sxv or sxvi, using fixed x-limits and variable y limits. + // inter_pts: intersections y_{j_tar-/+1/2}-yShift and y_{j_do-/+1/2} (lower/upper y-boundaries of donor cell). + // xc_do: cell center of donor cell. + // xc_tar: cell center of target cell. + // cellb_do: boundaries of target cell. + // cellb_tar: boundaries of target cell. + // is_upper_shift_dir_cell: is the donor the upper cell in the shift dir? + // shift_c: DG coefficients of the shift. + // mat_do: current donor matrix. + + double xi_b[] = {-1.0, 1.0}; // Limits of xi integral. + + double shift_dir_bounds[] = {up->ts_grid.lower[up->shift_dir_in_ts_grid], + up->ts_grid.upper[up->shift_dir_in_ts_grid]}; + + double x_eval = xc_do[up->shear_dir_in_ts_grid]; + double shift; + + up->shift_func(0.0, (double[]){x_eval}, &shift, up->shift_func_ctx); + + double shifted_coord = cellb_tar[cellb_lo(up->shift_dir_in_ts_grid)] - shift; + shifted_coord = ts_wrap_to_range(shifted_coord, shift_dir_bounds[0], shift_dir_bounds[1], + is_upper_shift_dir_cell); + + struct ts_shift_coord_shifted_log_ctx eta_lims_ctx = { + .shift_sign_fac = 1, + .xc_do = xc_do, + .xc_tar = xc_tar, + .dx = up->ts_grid.dx, + .pick_upper = is_upper_shift_dir_cell, + .shear_dir = up->shear_dir_in_ts_grid, + .shift_dir = up->shift_dir_in_ts_grid, + .shift_dir_bounds = {up->ts_grid.lower[up->shift_dir_in_ts_grid], + up->ts_grid.upper[up->shift_dir_in_ts_grid]}, + .shift_func = up->shift_func, + .shift_func_ctx = up->shift_func_ctx, + }; + + evalf_t eta_lims[2]; // Table of functions definting the limits of eta integral. + if ( cellb_do[cellb_lo(up->shift_dir_in_ts_grid)] <= shifted_coord && + shifted_coord <= cellb_do[cellb_up(up->shift_dir_in_ts_grid)] ) { + // sxv integral. + eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_lo(up->shift_dir_in_ts_grid)]; + eta_lims[0] = ts_shift_coord_shifted_log; + eta_lims[1] = ts_one; + } + else { + // sxvi integral. + eta_lims_ctx.shift_coord_tar = cellb_tar[cellb_up(up->shift_dir_in_ts_grid)]; + eta_lims[0] = ts_minus_one; + eta_lims[1] = ts_shift_coord_shifted_log; + } + + double etalo_xi[up->shift_b.num_basis], etaup_xi[up->shift_b.num_basis]; + ts_nod2mod_proj_1d(up, eta_lims[0], &eta_lims_ctx, xi_b, etalo_xi); + ts_nod2mod_proj_1d(up, eta_lims[1], &eta_lims_ctx, xi_b, etaup_xi); + // Offset between cell centers in direction of the shift. + double xs_off = ts_donor_target_offset(up, xc_do, xc_tar); + + if (fabs(xi_b[1] - xi_b[0]) > tol_xi) + up->kernels->ylimdg(1.0, xi_b[0], xi_b[1], etalo_xi, etaup_xi, + up->ts_grid.dx[up->shift_dir_in_ts_grid], xs_off, shift_c, mat_do); +} + +struct gkyl_nmat * +ts_calc_mats(struct gkyl_twistshift_dg *up) +{ + + // Allocate matrices containing the discrete subcell integrals. + int num_do_tot = 0; + for (int i=0; ishear_r.volume; i++) + num_do_tot += up->num_do[i]; + + struct gkyl_nmat *matsdo = gkyl_nmat_new(num_do_tot, up->basis.num_basis, up->basis.num_basis); + for (int n=0; nnum; ++n) { + struct gkyl_mat mat = gkyl_nmat_get(matsdo, n); + for (int j=0; jnc; ++j) + for (int i=0; inr; ++i) + gkyl_mat_set(&mat, i, j, 0.0); + } + + // y-index of the reference target used to precalc matrices. For positive(negative) + // yShift idx=1(last) might be better, but ideally it shouldn't matter. + int shift_dir_idx_tar = 1; + + double shift_dir_lims[] = {up->ts_grid.lower[up->shift_dir_in_ts_grid], + up->ts_grid.upper[up->shift_dir_in_ts_grid]}; + + // Create an eval_on_nodes updater to use its nodes and functions (but not + // the whole advance method). + up->ev_on_nod1d = gkyl_eval_on_nodes_new(&up->shear_grid, &up->shift_b, 1, ts_one, NULL); + // Create an array to store evaluations of a function at 1D nodes. + up->func_nod1d = gkyl_array_new(GKYL_DOUBLE, 1, up->shift_b.num_basis); + + struct gkyl_range_iter iter; + gkyl_range_iter_init(&iter, &up->shear_r); + while (gkyl_range_iter_next(&iter)) { + + // Get the cell boundaries and cell center. + int idx_tar[2]; // Target index. + idx_tar[up->shift_dir_in_ts_grid] = shift_dir_idx_tar; + idx_tar[up->shear_dir_in_ts_grid] = iter.idx[0]; + double cellb_tar[4] = {0.0}; // Cell boundaries, x lo and up, y lo and up; + double xc_tar[2] = {0.0}; // Cell center. + ts_grid_cell_boundaries(&up->ts_grid, idx_tar, cellb_tar); + gkyl_rect_grid_cell_center(&up->ts_grid, idx_tar, xc_tar); + + long shift_loc = gkyl_range_idx(&up->shear_r, iter.idx); + double *shift_c = (double *) gkyl_array_fetch(up->shift_dg, shift_loc); + + long linidx_do = ts_shift_dir_idx_do_linidx(up->num_do, iter.idx[0], shift_dir_idx_tar, + up->ts_grid.cells[up->shift_dir_in_ts_grid], up->shear_r.lower[0]); + int *shift_dir_idx_do_ptr = &up->shift_dir_idx_do[linidx_do]; + + long linidx_mats_do = 0; + for (int i=0; ishear_r.lower[0]; i++) + linidx_mats_do += up->num_do[i]; + + // Check that the shift variation within this x-cell < Ly. + // The algorithm assumes at most one x-intersection per (y-boundary, y-boundary) pair, + // which breaks when |S(x_up) - S(x_lo)| >= Ly. + double x_lo = cellb_tar[cellb_lo(up->shear_dir_in_ts_grid)]; + double x_up = cellb_tar[cellb_up(up->shear_dir_in_ts_grid)]; + double Ly = shift_dir_lims[1] - shift_dir_lims[0]; + double S_lo, S_up, S_c; + up->shift_func(0.0, (double[]){x_lo}, &S_lo, up->shift_func_ctx); + up->shift_func(0.0, (double[]){x_up}, &S_up, up->shift_func_ctx); + up->shift_func(0.0, (double[]){xc_tar[up->shear_dir_in_ts_grid]}, &S_c, up->shift_func_ctx); + + if (fabs(S_up - S_lo) >= Ly) { + fprintf(stderr, "bc_twistshift: shift variation |S(x_up)-S(x_lo)| = %g across a single x-cell" + " exceeds Ly = %g (cell ix=%d). Increase Nx, reduce the shear, or increase Ly.\n", + fabs(S_up - S_lo), Ly, iter.idx[0]); + assert(false); + } + + for (int iC=0; iCnum_do[iter.idx[0]-up->shear_r.lower[0]]; iC++){ + int idx_do[2]; // Target index. + idx_do[up->shift_dir_in_ts_grid] = shift_dir_idx_do_ptr[iC]; + idx_do[up->shear_dir_in_ts_grid] = iter.idx[0]; + + double cellb_do[4] = {0.0}; // Cell boundaries, x lo and up, y lo and up; + double xc_do[2] = {0.0}; // Cell center. + ts_grid_cell_boundaries(&up->ts_grid, idx_do, cellb_do); + gkyl_rect_grid_cell_center(&up->ts_grid, idx_do, xc_do); + + // Get the matrix we are presently assigning. + struct gkyl_mat mat_do = gkyl_nmat_get(matsdo, linidx_mats_do+iC); + + // Periodic copy in which to find the target for this donor-target pair: the integer nP such that + // S_c \approx y_tar_c - y_do_c + nP*Ly. All 4 inter_pts must use this same nP so that only + // roots from the physical intersection are accepted (not roots from other periodic copies). + int nP_primary = (int)round((S_c - (xc_tar[up->shift_dir_in_ts_grid] - xc_do[up->shift_dir_in_ts_grid])) / Ly); + + // Find the points where y_{j_tar-/+1/2}-yShift intersect the y=y_{j_do-/+1/2} lines. + // Also record the number and indices of points found/not found. + struct ts_val_found inter_pts[4] = {}; + int num_inter_pts_found = 0, num_inter_pts_not_found = 4; + int inter_pts_found_idxs[4], inter_pts_not_found_idxs[4]; + for (int i=0; i<2; i++) { // Loop over j_tar-/+1/2 + for (int j=0; j<2; j++) { // Loop over j_do-/+1/2 + double shift_dir_coord_tar = cellb_tar[2*up->shift_dir_in_ts_grid+i]; + double shift_dir_coord_do = cellb_do[2*up->shift_dir_in_ts_grid+j]; + struct gkyl_qr_res inter_res = ts_find_intersect(up, shift_dir_coord_tar, shift_dir_coord_do, + (double[]) {cellb_tar[cellb_lo(up->shear_dir_in_ts_grid)],cellb_tar[cellb_up(up->shear_dir_in_ts_grid)]}, + shift_dir_lims, nP_primary); + int ip_linc = i*2+j; + inter_pts[ip_linc].status = inter_res.status == 0; + if (inter_res.status == 0) { + inter_pts[ip_linc].value = inter_res.res; + inter_pts_found_idxs[num_inter_pts_found] = ip_linc; + num_inter_pts_found++; + } + else { + inter_pts_not_found_idxs[num_inter_pts_not_found] = ip_linc; + num_inter_pts_not_found--; + } + } + } + + bool is_upper_shift_dir_cell = idx_do[up->shift_dir_in_ts_grid] == up->ts_grid.cells[up->shift_dir_in_ts_grid]; + + if (num_inter_pts_found == 4) { + // sN: all intersections are found at this cell. + ts_subcellint_sNi_sNii(up, inter_pts, xc_do, xc_tar, cellb_do, cellb_tar, is_upper_shift_dir_cell, shift_c, &mat_do); + } + else if (num_inter_pts_found == 1) { + if (inter_pts[1].status) { + // si: y_{j_tar-1/2}-yShift intersects x_{i-1/2}. + // sii: y_{j_tar-1/2}-yShift intersects x_{i+1/2}. + ts_subcellint_si_sii(up, inter_pts, xc_do, xc_tar, cellb_do, cellb_tar, is_upper_shift_dir_cell, shift_c, &mat_do); + } + else { + // siii: y_{j_tar+1/2}-yShift intersects x_{i-1/2}. + // siv: y_{j_tar+1/2}-yShift intersects x_{i+1/2}. + ts_subcellint_siii_siv(up, inter_pts, xc_do, xc_tar, cellb_do, cellb_tar, is_upper_shift_dir_cell, shift_c, &mat_do); + } + } + else if (num_inter_pts_found == 3) { + if (!inter_pts[2].status) { + // sv: y_{j_tar+1/2}-yShift doesn't intersect y_{j_do-1/2} & intersects x_{i-1/2}. + // svi: y_{j_tar+1/2}-yShift doesn't intersect y_{j_do-1/2} & intersects x_{i+1/2}. + ts_subcellint_sv_svi(up, inter_pts, xc_do, xc_tar, cellb_do, cellb_tar, is_upper_shift_dir_cell, shift_c, &mat_do); + } + else { + // svii: y_{j_tar-1/2}-yShift doesn't intersect y_{j_do+1/2} & intersects x_{i-1/2}. + // sviii: y_{j_tar-1/2}-yShift doesn't intersect y_{j_do+1/2} & intersects x_{i+1/2}. + ts_subcellint_svii_sviii(up, inter_pts, xc_do, xc_tar, cellb_do, cellb_tar, is_upper_shift_dir_cell, shift_c, &mat_do); + } + } + else if (num_inter_pts_found == 2) { + if (inter_pts[0].status && inter_pts[1].status) { + // six: y_{j_tar-1/2}-yShift crosses y_{j_do-/+1/2} (increasing yShift). + // sx: y_{j_tar-1/2}-yShift crosses y_{j_do-/+1/2} (decreasing yShift). + ts_subcellint_six_sx(up, inter_pts, xc_do, xc_tar, cellb_do, cellb_tar, is_upper_shift_dir_cell, shift_c, &mat_do); + } + else if (inter_pts[2].status && inter_pts[3].status) { + // sxi: y_{j_tar+1/2}-yShift crosses y_{j_do-/+1/2} (decreasing yShift). + // sxii: y_{j_tar+1/2}-yShift crosses y_{j_do-/+1/2} (increasing yShift). + ts_subcellint_sxi_sxii(up, inter_pts, xc_do, xc_tar, cellb_do, cellb_tar, is_upper_shift_dir_cell, shift_c, &mat_do); + } + else { + // sxiii: y_{j_tar-1/2}-yShift crosses y_{j_do-1/2} & y_{j_tar+1/2}-yShift crosses y_{j_do+1/2} (increasing yShift). + // sxiv: y_{j_tar-1/2}-yShift crosses y_{j_do-1/2} & y_{j_tar+1/2}-yShift crosses y_{j_do+1/2} (decreasing yShift). + ts_subcellint_sxiii_sxiv(up, inter_pts, xc_do, xc_tar, cellb_do, cellb_tar, is_upper_shift_dir_cell, shift_c, &mat_do); + } + } + else if (num_inter_pts_found == 0) { + // sxv: y_{j_tar-1/2}-yShift crosses x_{i-/+1/2}. + // sxvi: y_{j_tar+1/2}-yShift crosses x_{i-/+1/2}. + ts_subcellint_sxv_sxvi(up, inter_pts, xc_do, xc_tar, cellb_do, cellb_tar, is_upper_shift_dir_cell, shift_c, &mat_do); + } + else { + // An error occurred. This shouldn't happen. + assert(false); + } + } + + } + + gkyl_array_release(up->func_nod1d); + gkyl_eval_on_nodes_release(up->ev_on_nod1d); + + struct gkyl_nmat *matsdo_out = up->use_gpu? gkyl_nmat_cu_dev_new(matsdo->num, matsdo->nr, matsdo->nc) + : gkyl_nmat_acquire(matsdo); + gkyl_nmat_copy(matsdo_out, matsdo); + gkyl_nmat_release(matsdo); + + return matsdo_out; +} + +long * +ts_calc_num_numcol_fidx_do(struct gkyl_twistshift_dg *up) +{ + // Calculate the linear indices into the donor distribution function gkyl_array + // for each num-numcol plane (in the num-numcol-num_basis) space. + + long *num_numcol_fidx_do_ho = (long*) gkyl_malloc(up->fmat->num * up->fmat->nc * sizeof(long)); + + // Location in the direction of the BC from which to take donor + // distributions. We assume that the user filled the ghost cell with the skin + // on the other side (i.e. applied periodicity first). + int bc_dir_loc_do = up->edge == GKYL_LOWER_EDGE? up->local_bcdir_ext_r.lower[up->bc_dir] + : up->local_bcdir_ext_r.upper[up->bc_dir]; + + // Range over directions other than shear and bc dirs. + struct gkyl_range shearbc_perp_r; + int remove[GKYL_MAX_DIM] = {0}, loc_in_dir[GKYL_MAX_DIM] = {0};; + remove[up->shear_dir] = remove[up->bc_dir] = 1; + loc_in_dir[up->shear_dir] = up->local_bcdir_ext_r.lower[up->shear_dir]; + loc_in_dir[up->bc_dir] = bc_dir_loc_do; + gkyl_range_deflate(&shearbc_perp_r, &up->local_bcdir_ext_r, remove, loc_in_dir); + + int shift_dir_in_shearbc_perp_r; + if (up->shift_dir < up->shear_dir && up->shift_dir < up->bc_dir) + shift_dir_in_shearbc_perp_r = up->shift_dir; + else if (up->shift_dir > up->shear_dir && up->shift_dir > up->bc_dir) + shift_dir_in_shearbc_perp_r = up->shift_dir-2; + else + shift_dir_in_shearbc_perp_r = up->shift_dir-1; + + int prev_shift_dir_idx = 0; + int donor_count = 0; + int do_idx[up->local_bcdir_ext_r.ndim]; + + // Loop over directions perpendicular to shear and BC dirs. + struct gkyl_range_iter iter; + gkyl_range_iter_init(&iter, &shearbc_perp_r); + while (gkyl_range_iter_next(&iter)) { + int ic = 0; + for (int d=0; dlocal_bcdir_ext_r.ndim; d++) { + if (d != up->bc_dir && d != up->shear_dir) { + do_idx[d] = iter.idx[ic]; + ic++; + } + } + do_idx[up->bc_dir] = bc_dir_loc_do; + + struct gkyl_range_iter shear_dir_iter; + gkyl_range_iter_init(&shear_dir_iter, &up->shear_r); + while (gkyl_range_iter_next(&shear_dir_iter)) { + + int shear_dir_idx = shear_dir_iter.idx[0]; + + long linidx_do = ts_shift_dir_idx_do_linidx(up->num_do, shear_dir_idx, + iter.idx[shift_dir_in_shearbc_perp_r], up->ts_grid.cells[up->shift_dir_in_ts_grid], up->shear_r.lower[0]); + + for (int i = 0; i < up->num_do[shear_dir_idx-up->shear_r.lower[0]]; i++) { + do_idx[up->shear_dir] = shear_dir_idx; + do_idx[up->shift_dir] = up->shift_dir_idx_do[linidx_do+i]; + + long loc = gkyl_range_idx(&up->local_bcdir_ext_r, do_idx); + num_numcol_fidx_do_ho[donor_count] = loc; + + donor_count += 1; + } + } + prev_shift_dir_idx = iter.idx[shift_dir_in_shearbc_perp_r]; + } + + long *num_numcol_fidx_do; + if (!up->use_gpu) { + num_numcol_fidx_do = (long*) gkyl_malloc(up->fmat->num * up->fmat->nc * sizeof(long)); + memcpy(num_numcol_fidx_do, num_numcol_fidx_do_ho, up->fmat->num * up->fmat->nc * sizeof(long)); + } +#ifdef GKYL_HAVE_CUDA + if (up->use_gpu) { + num_numcol_fidx_do = (long*) gkyl_cu_malloc(up->fmat->num * up->fmat->nc * sizeof(long)); + gkyl_cu_memcpy(num_numcol_fidx_do, num_numcol_fidx_do_ho, up->fmat->num * up->fmat->nc * sizeof(long), GKYL_CU_MEMCPY_H2D); + } +#endif + + gkyl_free(num_numcol_fidx_do_ho); + + return num_numcol_fidx_do; +} + +long * +ts_calc_num_numcol_fidx_tar(struct gkyl_twistshift_dg *up) +{ + + long *num_numcol_fidx_tar_ho = (long*) gkyl_malloc(up->fmat->num * up->fmat->nc * sizeof(long)); + + // Location in the direction of the BC in which to place the target + // distributions. We assume that the local_bcdir_ext_r is a range extended in z (it + // includes the z ghose cell). + int bc_dir_loc_tar = up->edge == GKYL_LOWER_EDGE? up->local_bcdir_ext_r.lower[up->bc_dir] + : up->local_bcdir_ext_r.upper[up->bc_dir]; + + // Range over directions other than shear and bc dirs. + struct gkyl_range shearbc_perp_r; + int remove[GKYL_MAX_DIM] = {0}, loc_in_dir[GKYL_MAX_DIM] = {0};; + remove[up->shear_dir] = remove[up->bc_dir] = 1; + loc_in_dir[up->shear_dir] = up->local_bcdir_ext_r.lower[up->shear_dir]; + loc_in_dir[up->bc_dir] = bc_dir_loc_tar; + gkyl_range_deflate(&shearbc_perp_r, &up->local_bcdir_ext_r, remove, loc_in_dir); + + int tar_idx[up->local_bcdir_ext_r.ndim]; + int tar_count = 0; + + // Loop over directions perpendicular to shear and BC dirs. + struct gkyl_range_iter iter; + gkyl_range_iter_init(&iter, &shearbc_perp_r); + while (gkyl_range_iter_next(&iter)) { + + int ic = 0; + for (int d=0; dlocal_bcdir_ext_r.ndim; d++) { + if (d != up->bc_dir && d != up->shear_dir) { + tar_idx[d] = iter.idx[ic]; + ic++; + } + } + tar_idx[up->bc_dir] = bc_dir_loc_tar; + + + struct gkyl_range_iter shear_dir_iter; + gkyl_range_iter_init(&shear_dir_iter, &up->shear_r); + while (gkyl_range_iter_next(&shear_dir_iter)) { + int shear_dir_idx = shear_dir_iter.idx[0]; + tar_idx[up->shear_dir] = shear_dir_idx; + + long loc = gkyl_range_idx(&up->local_bcdir_ext_r, tar_idx); + num_numcol_fidx_tar_ho[tar_count] = loc; + tar_count += 1; + } + } + + long *num_numcol_fidx_tar; + if (!up->use_gpu) { + num_numcol_fidx_tar = (long*) gkyl_malloc(up->fmat->num * up->fmat->nc * sizeof(long)); + memcpy(num_numcol_fidx_tar, num_numcol_fidx_tar_ho, up->fmat->num * up->fmat->nc * sizeof(long)); + } +#ifdef GKYL_HAVE_CUDA + if (up->use_gpu) { + num_numcol_fidx_tar = (long*) gkyl_cu_malloc(up->fmat->num * up->fmat->nc * sizeof(long)); + gkyl_cu_memcpy(num_numcol_fidx_tar, num_numcol_fidx_tar_ho, up->fmat->num * up->fmat->nc * sizeof(long), GKYL_CU_MEMCPY_H2D); + } +#endif + + gkyl_free(num_numcol_fidx_tar_ho); + + return num_numcol_fidx_tar; +} + +void +gkyl_twistshift_dg_choose_kernels(struct gkyl_basis basis, int cdim, int shift_poly_order, + struct gkyl_twistshift_dg_kernels *kers) +{ + int dim = basis.ndim; + int vdim = dim - cdim; + enum gkyl_basis_type basis_type = basis.b_type; + int poly_order = basis.poly_order; + switch (basis_type) { + case GKYL_BASIS_MODAL_GKHYBRID: + case GKYL_BASIS_MODAL_SERENDIPITY: + if (shift_poly_order == 1) { + kers->xlimdg = vdim==0? ser_twistshift_xlimdg_list_0v_yShp1[cdim-2].kernels[poly_order] + : ser_twistshift_xlimdg_list_2v_yShp1[cdim-2].kernels[poly_order]; + kers->ylimdg = vdim==0? ser_twistshift_ylimdg_list_0v_yShp1[cdim-2].kernels[poly_order] + : ser_twistshift_ylimdg_list_2v_yShp1[cdim-2].kernels[poly_order]; + kers->fullcell = vdim==0? ser_twistshift_fullcell_list_0v_yShp1[cdim-2].kernels[poly_order] + : ser_twistshift_fullcell_list_2v_yShp1[cdim-2].kernels[poly_order]; + } + else if (shift_poly_order == 2) { + assert(false); // MF 2025/09/20: removed 3x2v kernel because it's 8.5 MB. + kers->xlimdg = vdim==0? ser_twistshift_xlimdg_list_0v_yShp2[cdim-2].kernels[poly_order] + : ser_twistshift_xlimdg_list_2v_yShp2[cdim-2].kernels[poly_order]; + kers->ylimdg = vdim==0? ser_twistshift_ylimdg_list_0v_yShp2[cdim-2].kernels[poly_order] + : ser_twistshift_ylimdg_list_2v_yShp2[cdim-2].kernels[poly_order]; + kers->fullcell = vdim==0? ser_twistshift_fullcell_list_0v_yShp2[cdim-2].kernels[poly_order] + : ser_twistshift_fullcell_list_2v_yShp2[cdim-2].kernels[poly_order]; + } + return; + default: + assert(false); + break; + } +} + +struct gkyl_twistshift_dg* +gkyl_twistshift_dg_inew(const struct gkyl_twistshift_dg_inp *inp) +{ + + // Allocate space for new updater. + struct gkyl_twistshift_dg *up = gkyl_malloc(sizeof(struct gkyl_twistshift_dg)); + + up->bc_dir = inp->bc_dir; + up->shift_dir = inp->shift_dir; + up->shear_dir = inp->shear_dir; + up->edge = inp->edge; + up->basis = *inp->basis; + up->grid = *inp->grid; + up->use_gpu = inp->use_gpu; + up->local_bcdir_ext_r = *inp->bcdir_ext_update_r; + + // Assume the poly order of the DG shift is the same as that of the field, + // unless requested otherwise. + up->shift_poly_order = inp->basis->poly_order; + if (inp->shift_poly_order) + up->shift_poly_order = inp->shift_poly_order; + + const int ndim = inp->bcdir_ext_update_r->ndim; + // Check that it is being used for 3D or 5D. Likely only small changes are + // needed to make it work in other dimensions. + assert(ndim == 3 || ndim == 5); + + double lo1d[1], up1d[1]; int cells1d[1]; + + // Create 1D grid and range in the direction of the shear. + gkyl_range_init(&up->shear_r, 1, (int[]) {up->local_bcdir_ext_r.lower[inp->shear_dir]}, + (int[]) {up->local_bcdir_ext_r.upper[inp->shear_dir]}); + lo1d[0] = inp->grid->lower[up->shear_dir]; + up1d[0] = inp->grid->upper[up->shear_dir]; + cells1d[0] = inp->grid->cells[up->shear_dir]; + gkyl_rect_grid_init(&up->shear_grid, 1, lo1d, up1d, cells1d); + int idx[] = {up->shear_r.lower[0]}; + long linidx = gkyl_range_idx(&up->shear_r, idx); + + // Create 1D grid and range in the diretion of the shift. + gkyl_range_init(&up->shift_r, 1, (int[]) {up->local_bcdir_ext_r.lower[inp->shift_dir]}, + (int[]) {up->local_bcdir_ext_r.upper[inp->shift_dir]}); + lo1d[0] = inp->grid->lower[up->shift_dir]; + up1d[0] = inp->grid->upper[up->shift_dir]; + cells1d[0] = inp->grid->cells[up->shift_dir]; + gkyl_rect_grid_init(&up->shift_grid, 1, lo1d, up1d, cells1d); + + // Create 2D grid (and range) the twist-shift takes place in. + int dimlo, dimup; + if (up->shift_dir < up->shear_dir) { + dimlo = up->shift_dir; + dimup = up->shear_dir; + up->shift_dir_in_ts_grid = 0; + up->shear_dir_in_ts_grid = 1; + } + else { + dimlo = up->shear_dir; + dimup = up->shift_dir; + up->shift_dir_in_ts_grid = 1; + up->shear_dir_in_ts_grid = 0; + } + gkyl_range_init(&up->ts_r, 2, (int[]) {up->local_bcdir_ext_r.lower[dimlo], up->local_bcdir_ext_r.lower[dimup]}, + (int[]) {up->local_bcdir_ext_r.upper[dimlo], up->local_bcdir_ext_r.upper[dimup]}); + double lo2d[] = {inp->grid->lower[dimlo], inp->grid->lower[dimup]}; + double up2d[] = {inp->grid->upper[dimlo], inp->grid->upper[dimup]}; + int cells2d[] = {inp->grid->cells[dimlo], inp->grid->cells[dimup]}; + gkyl_rect_grid_init(&up->ts_grid, 2, lo2d, up2d, cells2d); + + // Project the shift onto the shift basis. + gkyl_cart_modal_serendip(&up->shift_b, 1, up->shift_poly_order); + if (inp->shift_func) { + up->shift_dg = gkyl_array_new(GKYL_DOUBLE, up->shift_b.num_basis, up->shear_r.volume); + gkyl_eval_on_nodes *evup = gkyl_eval_on_nodes_new(&up->shear_grid, &up->shift_b, 1, + inp->shift_func, inp->shift_func_ctx); + gkyl_eval_on_nodes_advance(evup, 0.0, &up->shear_r, up->shift_dg); + gkyl_eval_on_nodes_release(evup); + } + else { + // The shift must be discretized on the shear cells this updater indexes. + assert(inp->shift_dg->size == up->shear_r.volume); + assert(inp->shift_dg->ncomp == up->shift_b.num_basis); + up->shift_dg = gkyl_array_acquire(inp->shift_dg); + } + + // Function defining the shift (and its context). + if (shift_func_op == 0) { + up->shift_func = ts_shift_dg_eval; + up->shift_dg_eval_ctx.shift_dg = up->shift_dg; + up->shift_dg_eval_ctx.shift_b = &up->shift_b; + up->shift_dg_eval_ctx.shear_grid = &up->shear_grid; + up->shift_dg_eval_ctx.shear_r = &up->shear_r; + up->shift_func_ctx = &up->shift_dg_eval_ctx; + } + else if (shift_func_op == 1) { + up->shift_func = inp->shift_func; + up->shift_func_ctx = inp->shift_func_ctx; + } + else { + fprintf(stderr, "Twist-shift function option not recognized. Exiting...\n"); + assert(false); + } + + // Find the donor cells for each target. Store the number of donors for each + // shear_dir idx (num_do) & the shift_dir idx of each donor (shift_dir_idx_do). + // i.e. allocates and assigns num_do and up->shift_dir_idx_do. + ts_find_donors(up); + + // Array of cummulative number of donors at given shear_dir cell. + const int num_do_cum_sz = up->grid.cells[up->shear_dir]+1; + int num_do_cum_ho[num_do_cum_sz]; + for (int i=0; ishear_r.lower[0]; ishear_r.upper[0]+1; i++) + num_do_cum_ho[i] = num_do_cum_ho[i-1] + up->num_do[i-up->shear_r.lower[0]]; + + if (!up->use_gpu) { + up->num_do_cum = gkyl_malloc(num_do_cum_sz * sizeof(int)); + memcpy(up->num_do_cum, num_do_cum_ho, num_do_cum_sz * sizeof(int)); + } +#ifdef GKYL_HAVE_CUDA + if (up->use_gpu) { + up->num_do_cum = gkyl_cu_malloc(num_do_cum_sz * sizeof(int)); + gkyl_cu_memcpy(up->num_do_cum, num_do_cum_ho, num_do_cum_sz * sizeof(int), GKYL_CU_MEMCPY_H2D); + } +#endif + + // Choose the kernels that do the subcell and full cell integrals + up->kernels = gkyl_malloc(sizeof(struct gkyl_twistshift_dg_kernels)); + gkyl_twistshift_dg_choose_kernels(*inp->basis, inp->cdim, up->shift_poly_order, up->kernels); + + // The BC is applied as a set of matrix-matrix multiplications + // f_i = sum_{q}^{N_do(i)} A_q,i B_q,i + // where i indicates the shear_dir cell index, A_q is a + // num_basis x num_basis matrix containing the discretization + // of subcell integrals, B_q is a num_basis x (Ny * Nvpar * Nmu) matrix with + // the DG coefficients of f common to a given A_q matrix, and thus where f_i + // is a num_basis x (Ny*Nvpar*Nmu) matrix. + // + // Naming scheme: + // A_q: scimat (subscell integral matrices). + // B_q: fmat (distribution function matrices). + // A_q . B_q: mm_contr (contributions from mat-mat multiplication). + + // Calculate the entries in the matrices used to apply the BC. + up->scimat = ts_calc_mats(up); + + // Number of colums in fmat. + int fmat_num_col = 1; + for (int d=0; dbc_dir && d != up->shear_dir) + fmat_num_col *= up->local_bcdir_ext_r.upper[d] - up->local_bcdir_ext_r.lower[d] + 1; + } + + if (!up->use_gpu) { + up->fmat = gkyl_nmat_new(up->scimat->num, up->scimat->nr, fmat_num_col); + up->mm_contr = gkyl_nmat_new(up->scimat->num, up->scimat->nr, fmat_num_col); + } +#ifdef GKYL_HAVE_CUDA + if (up->use_gpu) { + up->fmat = gkyl_nmat_cu_dev_new(up->scimat->num, up->scimat->nr, fmat_num_col); + up->mm_contr = gkyl_nmat_cu_dev_new(up->scimat->num, up->scimat->nr, fmat_num_col); + } +#endif + + // Index translation from num-numcol plane index to linear index into the + // donor distribution function gkyl_array. + up->num_numcol_fidx_do = ts_calc_num_numcol_fidx_do(up); + + // Index translation from num-numcol plane index to linear index into the + // tar distribution function gkyl_array. + up->num_numcol_fidx_tar = ts_calc_num_numcol_fidx_tar(up); + + // Permutted ghost range, for indexing into the target field. + // Order: Shift direction, redundant directions, shear direction. + int lo4D[ndim-1], up4D[ndim-1]; + lo4D[0] = up->local_bcdir_ext_r.lower[up->shift_dir]; + up4D[0] = up->local_bcdir_ext_r.upper[up->shift_dir]; + int ic = 1; + for (int d=0; dbc_dir && d != up->shear_dir && d != up->shift_dir) { + lo4D[ic] = up->local_bcdir_ext_r.lower[d]; + up4D[ic] = up->local_bcdir_ext_r.upper[d]; + ic++; + } + } + lo4D[ndim-2] = up->local_bcdir_ext_r.lower[up->shear_dir]; + up4D[ndim-2] = up->local_bcdir_ext_r.upper[up->shear_dir]; + gkyl_range_init(&up->permutted_ghost_r, ndim-1, lo4D, up4D); + + // Create a ghost range, to clear it before adding contributions from TS BC. + if (inp->edge == GKYL_LOWER_EDGE) + gkyl_range_shorten_from_above(&up->ghost_r, &up->local_bcdir_ext_r, inp->bc_dir, inp->num_ghost[inp->bc_dir]); + else + gkyl_range_shorten_from_below(&up->ghost_r, &up->local_bcdir_ext_r, inp->bc_dir, inp->num_ghost[inp->bc_dir]); + + return up; +} + +struct gkyl_twistshift_dg* +gkyl_twistshift_dg_new(int bc_dir, int shift_dir, int shear_dir, + enum gkyl_edge_loc edge, int cdim, const struct gkyl_range *bcdir_ext_update_r, const int *num_ghost, + const struct gkyl_basis *basis, const struct gkyl_rect_grid *grid, evalf_t shift_func, void *shift_func_ctx, + struct gkyl_array *shift_dg, int shift_poly_order, bool use_gpu) +{ + struct gkyl_twistshift_dg_inp inp = { + .bc_dir = bc_dir , + .shift_dir = shift_dir , + .shear_dir = shear_dir , + .edge = edge , + .cdim = cdim , + .bcdir_ext_update_r = bcdir_ext_update_r, + .num_ghost = num_ghost , + .basis = basis , + .grid = grid , + .shift_func = shift_func , + .shift_func_ctx = shift_func_ctx , + .shift_dg = shift_dg , + .use_gpu = use_gpu , + .shift_poly_order = shift_poly_order , + }; + return gkyl_twistshift_dg_inew(&inp); +} + +void +gkyl_twistshift_dg_advance(struct gkyl_twistshift_dg *up, struct gkyl_array *fdo, struct gkyl_array *ftar) +{ + +#ifdef GKYL_HAVE_CUDA + if (up->use_gpu) { + gkyl_twistshift_dg_advance_cu(up, fdo, ftar); + return; + } +#endif + + // Assign the distribution matrices. + // This assumes that fdo->ncomp = fmat->nr. + // Recall: + // fmat->num = sum_i^Nx num_do(i) + // fmat->nr = num_basis = ncomp + // fmat->nc = Ny*Nvpar*Nmu + for (size_t i=0; ifmat->num * up->fmat->nr * up->fmat->nc; i++) { + + long nc_idx = i / up->fmat->nr; // num-num_col index: current num-num_col plane. + int row_idx = i % up->fmat->nr; // row index: current DG coeff. + + // This if-statement may only be needed in GPU kernel, not for CPUs. + if ((nc_idx < up->fmat->num * up->fmat->nc) && (row_idx < fdo->ncomp)) { + const double *fdo_c = (const double*) gkyl_array_cfetch(fdo, up->num_numcol_fidx_do[nc_idx]); + struct gkyl_mat mcurr = gkyl_nmat_get(up->fmat, nc_idx % up->fmat->num); + + gkyl_mat_set(&mcurr, row_idx, nc_idx/up->fmat->num, fdo_c[row_idx]); + } + } + + // Perform the mat-mat multiplications. + gkyl_nmat_mm(1.0, 0.0, GKYL_NO_TRANS, up->scimat, GKYL_NO_TRANS, up->fmat, up->mm_contr); + + // Clear the ghost range. + gkyl_array_clear_range(ftar, 0.0, &up->ghost_r); + + // Perform reduction over num_do contributions from mat-mat mults (mm_contr). + int num_cells_skin = (up->shear_r.upper[0]-up->shear_r.lower[0]+1) * up->fmat->nc; + for (size_t i=0; incomp * num_cells_skin; i++) { + + long linidx_tar = i / ftar->ncomp; + int row_idx = i % ftar->ncomp; + + // This if-statement may only be needed in GPU kernel, not for CPUs. + if ((linidx_tar < num_cells_skin) && (row_idx < ftar->ncomp)) { + double *ftar_c = (double*) gkyl_array_fetch(ftar, up->num_numcol_fidx_tar[linidx_tar]); + + int idx[GKYL_MAX_DIM] = {1}; + gkyl_sub_range_inv_idx(&up->permutted_ghost_r, linidx_tar, idx); + + int ac[GKYL_MAX_DIM] = {1}; + for (int d=2; dgrid.ndim-1; d++) + ac[d-2] = up->grid.cells[d+1]; + ac[up->permutted_ghost_r.ndim-2] = up->mm_contr->num; + + int start = 0; + for (int d=0; dpermutted_ghost_r.ndim-1; d++) + start = (start + (idx[d]-1)) * ac[d]; + + int shear_idx = idx[up->permutted_ghost_r.ndim-1]; + + int do_start = up->num_do_cum[shear_idx-1]; + int do_end = up->num_do_cum[shear_idx-1+1]; + for (int j=do_start; jmm_contr, linidx_mm_contr % up->mm_contr->num); + ftar_c[row_idx] += gkyl_mat_get(&mat, row_idx, linidx_mm_contr / up->mm_contr->num); + } + } + } +} + +struct gkyl_array* +gkyl_twistshift_dg_get_shift_objects(struct gkyl_twistshift_dg *up, struct gkyl_rect_grid *shear_grid, + struct gkyl_range *shear_r, struct gkyl_basis *shift_b) +{ + *shear_grid = up->shear_grid; + *shear_r = up->shear_r ; + *shift_b = up->shift_b ; + return gkyl_array_acquire(up->shift_dg); +}; + +void +gkyl_twistshift_dg_release(struct gkyl_twistshift_dg *up) { + // Release memory associated with this updater. + if (!up->use_gpu) { + gkyl_free(up->num_do_cum); + gkyl_free(up->num_numcol_fidx_do); + gkyl_free(up->num_numcol_fidx_tar); + } +#ifdef GKYL_HAVE_CUDA + if (up->use_gpu) { + gkyl_cu_free(up->num_do_cum); + gkyl_cu_free(up->num_numcol_fidx_do); + gkyl_cu_free(up->num_numcol_fidx_tar); + } +#endif + + gkyl_nmat_release(up->fmat); + gkyl_nmat_release(up->mm_contr); + + gkyl_nmat_release(up->scimat); + + gkyl_free(up->kernels); + + gkyl_array_release(up->shift_dg); + + gkyl_free(up->num_do); + gkyl_free(up->shift_dir_idx_do); + + gkyl_free(up); +} diff --git a/gyrokinetic/zero/bc_twistshift_cu.cu b/gyrokinetic/zero/twistshift_dg_cu.cu similarity index 89% rename from gyrokinetic/zero/bc_twistshift_cu.cu rename to gyrokinetic/zero/twistshift_dg_cu.cu index f4cbbecab7..a9987ffc92 100644 --- a/gyrokinetic/zero/bc_twistshift_cu.cu +++ b/gyrokinetic/zero/twistshift_dg_cu.cu @@ -8,8 +8,8 @@ extern "C" { #include #include #include -#include -#include +#include +#include } #include @@ -19,7 +19,7 @@ extern "C" { #define START_ID (threadIdx.x + blockIdx.x*blockDim.x) __global__ void -gkyl_bc_twistshift_set_distf_mats_cu_ker(const struct gkyl_array *fdo, const long *num_numcol_fidx_do, struct gkyl_nmat *fmat) +gkyl_twistshift_dg_set_distf_mats_cu_ker(const struct gkyl_array *fdo, const long *num_numcol_fidx_do, struct gkyl_nmat *fmat) { // Assign the distribution matrices. // This assumes that fdo->ncomp = fmat->nr. @@ -40,7 +40,7 @@ gkyl_bc_twistshift_set_distf_mats_cu_ker(const struct gkyl_array *fdo, const lon } __global__ void -gkyl_bc_twistshift_add_contr_cu_ker(struct gkyl_array *ftar, long *num_numcol_fidx_tar, int num_cells_skin, +gkyl_twistshift_dg_add_contr_cu_ker(struct gkyl_array *ftar, long *num_numcol_fidx_tar, int num_cells_skin, struct gkyl_nmat *mm_contr, int *num_do_cum, struct gkyl_range permutted_ghost_r, struct gkyl_rect_grid grid) { long linidx_tar = START_ID / ftar->ncomp; @@ -75,11 +75,11 @@ gkyl_bc_twistshift_add_contr_cu_ker(struct gkyl_array *ftar, long *num_numcol_fi } void -gkyl_bc_twistshift_advance_cu(struct gkyl_bc_twistshift *up, struct gkyl_array *fdo, struct gkyl_array *ftar) +gkyl_twistshift_dg_advance_cu(struct gkyl_twistshift_dg *up, struct gkyl_array *fdo, struct gkyl_array *ftar) { // Set the columns of the donor matrix with the donor distributions. int num_blocks_set = (up->fmat->num * up->fmat->nr * up->fmat->nc+GKYL_DEFAULT_NUM_THREADS-1)/GKYL_DEFAULT_NUM_THREADS; - gkyl_bc_twistshift_set_distf_mats_cu_ker<<>> + gkyl_twistshift_dg_set_distf_mats_cu_ker<<>> (fdo->on_dev, up->num_numcol_fidx_do, up->fmat->on_dev); // Perform the mat-mat multiplications. @@ -91,7 +91,7 @@ gkyl_bc_twistshift_advance_cu(struct gkyl_bc_twistshift *up, struct gkyl_array * // Add the contributions of mat-vec multiplications. int num_cells_skin = (up->shear_r.upper[0]-up->shear_r.lower[0]+1) * up->fmat->nc; int num_blocks_add = (ftar->ncomp * num_cells_skin+GKYL_DEFAULT_NUM_THREADS-1)/GKYL_DEFAULT_NUM_THREADS; - gkyl_bc_twistshift_add_contr_cu_ker<<>> + gkyl_twistshift_dg_add_contr_cu_ker<<>> (ftar->on_dev, up->num_numcol_fidx_tar, num_cells_skin, up->mm_contr->on_dev, up->num_do_cum, up->permutted_ghost_r, up->grid); }