###
### Variables
###

# Global variables (will be set in every job):
variables:
  WARN_AS_ERROR: "1"
  # MAKEFLAGS: "-s"
  REALM_BACKTRACE: "1"
  REALM_SYNTHETIC_CORE_MAP: "" # Disable Realm thread pinning.
  TIMELIMIT: "600"  # each test should take less than 10 minutes
  USE_DEFCHECK: "1"
  CCACHE_BASEDIR: $CI_PROJECT_DIR
  CCACHE_DIR: $CI_PROJECT_DIR/ccache
  USE_CCACHE: "1"

cache: &global_cache
  key: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"
  paths:
    - $CCACHE_DIR


# Local variables (included on a case-by-case basis in each job):
.gcc8: &gcc8
  CC:  "gcc-8"
  CXX: "g++-8"
  F90: "gfortran-8"
.gcc9: &gcc9
  CC:  "gcc-9"
  CXX: "g++-9"
  F90: "gfortran-9"
.gcc10: &gcc10
  CC:  "gcc-10"
  CXX: "g++-10"
  F90: "gfortran-10"
.gcc11: &gcc11
  CC:  "gcc-11"
  CXX: "g++-11"
  F90: "gfortran-11"
.gcc12: &gcc12
  CC:  "gcc-12"
  CXX: "g++-12"
  F90: "gfortran-12"

.clang7: &clang7
  CC:  "clang-7"
  CXX: "clang++-7"
  F90: "gfortran-5"
.clang8: &clang8
  CC:  "clang-8"
  CXX: "clang++-8"
  F90: "gfortran-5"
.clang9: &clang9 # FIXME (Elliott): this configuration for kokkos_openmp only. do not use for any other test job
  CC:  "clang-9"
  CXX: "clang++-9"
  F90: "gfortran-9"
.clang10: &clang10
  CC:  "clang-10"
  CXX: "clang++-10"
  F90: "gfortran-7"
.clang12: &clang12
  CC:  "clang-12"
  CXX: "clang++-12"
  F90: "gfortran-9"
.clang14: &clang14
  CC:  "clang-14"
  CXX: "clang++-14"
  F90: "gfortran-11"
.hpcsdk2211: &hpcsdk2211
  CC: "/opt/nvidia/hpc_sdk/Linux_x86_64/22.11/compilers/bin/nvc"
  CXX: "/opt/nvidia/hpc_sdk/Linux_x86_64/22.11/compilers/bin/nvc++"
  F90: "/opt/nvidia/hpc_sdk/Linux_x86_64/22.11/compilers/bin/nvfortran"
  USE_CCACHE: "0"
  # cmake gets confused and can't figure out library arch
  # EXTRA_CMAKE_ARGS: "-DCMAKE_LIBRARY_ARCHITECTURE=x86_64-linux-gnu"
.icc212: &icc212
  CC: "/opt/intel/oneapi/compiler/2021.2.0/linux/bin/intel64/icc"
  CXX: "/opt/intel/oneapi/compiler/2021.2.0/linux/bin/intel64/icpc"
  F90: "/opt/intel/oneapi/compiler/2021.2.0/linux/bin/intel64/ifort"

.terra70: &terra70
  LLVM_CONFIG: "llvm-config-7"
  TERRA_DIR: "/usr/local/terra70"
.terra9: &terra9
  LLVM_CONFIG: "llvm-config-9"
  TERRA_DIR: "/usr/local/terra9"
.terra10: &terra10
  LLVM_CONFIG: "llvm-config-10"
  TERRA_DIR: "/usr/local/terra10"
.terra12: &terra12
  LLVM_CONFIG: "llvm-config-12"
  TERRA_DIR: "/usr/local/terra12"
.terra14: &terra14
  LLVM_CONFIG: "llvm-config-14"
  TERRA_DIR: "/usr/local/terra14"

.llvm7: &llvm7
  LLVM_CONFIG: "llvm-config-7"
.llvm9: &llvm9
  LLVM_CONFIG: "llvm-config-9"
.llvm12: &llvm12
  LLVM_CONFIG: "llvm-config-12"
.llvm14: &llvm14
  LLVM_CONFIG: "llvm-config-14"

.debug: &debug
  DEBUG: "1"
.release: &release
  DEBUG: "0"

.maxdim4: &maxdim4
  MAX_DIM: "4"      # test up to 4-D (needed for HTR external test)
.maxdim5: &maxdim5
  MAX_DIM: "5"      # test up to 5-D (useful because 1-4 have special cases)

.spy: &spy
  USE_SPY: "1"
  BASH_ENV: "/root/.cargo/env" # make sure Cargo environment gets loaded

.checks: &checks
  BOUNDS_CHECKS: "1"
  PRIVILEGE_CHECKS: "1"

.cxx11_normal: &cxx11_normal
  CXXFLAGS: ""
  CXX_STANDARD: "11"
.cxx11_hardened: &cxx11_hardened
  CXXFLAGS: "-D_FORTIFY_SOURCE=2 -D_GLIBCXX_ASSERTIONS"
  CXX_STANDARD: "11"
.cxx14_normal: &cxx14_normal
  CXXFLAGS: ""
  CXX_STANDARD: "14"
.cxx17_normal: &cxx17_normal
  CXXFLAGS: ""
  CXX_STANDARD: "17"
.cxx20_normal: &cxx20_normal
  CXXFLAGS: ""
  CXX_STANDARD: "20"
.cxx11_32bit_normal: &cxx11_32bit_normal
  CXXFLAGS: "-m32"
  LDFLAGS: "-m32 -latomic"
  MARCH: "i686"
  CXX_STANDARD: "11"
.shared: &shared
  SHARED_OBJECTS: "1"

.openmp: &openmp
  USE_OPENMP: "1"
.python3: &python3
  USE_PYTHON: "1"
  # FIXME: It would be nice to find a better way to do this,
  # but the alternatives look scary
  PYTHON_EXE: "python3"
  PYTHON_LIB: "/usr/lib/x86_64-linux-gnu/libpython3.8.so"
  PYTHON_VERSION_MAJOR: "3"
  TEST_PYTHON_EXE: "python3"
.hdf5: &hdf5
  USE_HDF: "1"
  HDF_HEADER: "hdf5/serial/hdf5.h"
  HDF_LIBNAME: "hdf5_serial"
.fortran: &fortran
  LEGION_USE_FORTRAN: "1"
.llvm: &llvm
  USE_LLVM: "1"
.cuda: &cuda
  USE_CUDA: "1"
  USE_NVTX: "1"
.cuda_dynamic: &cuda_dynamic
  EXTRA_CMAKE_ARGS: "-DLegion_CUDA_DYNAMIC_LOAD=ON -DLegion_HIJACK_CUDART=OFF"
.gasnet1_mpi: &gasnet1_mpi
  REALM_NETWORKS: "gasnet1"
  CONDUIT: "mpi"
.gasnetex_mpi: &gasnetex_mpi
  REALM_NETWORKS: "gasnetex"
  CONDUIT: "mpi"
.gasnetex_mpi_debug: &gasnetex_mpi_debug
  REALM_NETWORKS: "gasnetex"
  CONDUIT: "mpi"
  GASNET_DEBUG: "1"
  # GASNet headers are not guaranteed to be warning-free in debug mode
  WARN_AS_ERROR: "0"
.gasnetex_mpi_nightly: &gasnetex_mpi_nightly
  REALM_NETWORKS: "gasnetex"
  CONDUIT: "mpi"
  GASNET_VERSION: "GASNet-EX-snapshot"
  # source URL is set by a GitLab CI variable as the GASNet-EX team prefers
  #  it not to be public at this time
  #GASNETEX_SNAPSHOT_SOURCE_URL: ...
.gasnetex_ucx : &gasnetex_ucx
  REALM_NETWORKS: "gasnetex"
  CONDUIT: "ucx"
  UCX_SOURCE: "https://github.com/openucx/ucx/releases/download/v1.8.0/ucx-1.8.0.tar.gz"
  UCX_WARN_UNUSED_ENV_VARS: "n"
.gasnet1_ucx : &gasnet1_ucx
  REALM_NETWORKS: "gasnet1"
  CONDUIT: "ucx"
  UCX_SOURCE: "https://github.com/openucx/ucx/releases/download/v1.8.0/ucx-1.8.0.tar.gz"
  UCX_WARN_UNUSED_ENV_VARS: "n"
.gasnet_embed_local : &gasnet_embed_local
  EMBED_GASNET: "1"
  EMBED_GASNET_LOCAL: "1"
.gasnet_embed_remote : &gasnet_embed_remote
  EMBED_GASNET: "1"
.mpi: &mpi
  REALM_NETWORKS: "mpi"
  # mpich-3.2
  MPIR_CVAR_CH3_NOLOCAL: "1"
  # mpich-3.3 & above
  MPIR_CVAR_NOLOCAL: "1"
.ucx: &ucx
  REALM_NETWORKS: "ucx"
  UCX_WARN_UNUSED_ENV_VARS: "n"
  # mpi_interop example requires MPI to build correctly
  USE_MPI: "1"
.cmake: &cmake
  USE_CMAKE: "1"
.kokkos: &kokkos
  USE_KOKKOS: "1"
  KOKKOS_REPO: "https://github.com/kokkos/kokkos.git"
  KOKKOS_BRANCH: "release-candidate-4.0.0"
  KOKKOS_COMMIT: "14deae4036aa7165c55fc53bf8634f654ac6e257"
  KOKKOS_CUDA_ARCH: "PASCAL60"
  KKERNELS_REPO: "https://github.com/kokkos/kokkos-kernels.git"
  KKERNELS_BRANCH: "develop"
  KKERNELS_COMMIT: "bce06df1aa1fc0634aecfb579eb41341db9c9bcf"

.prof: &prof
  USE_PROF: "1"
  BASH_ENV: "/root/.cargo/env" # make sure Cargo environment gets loaded

.legion: &legion
  TEST_REGENT: "0"
  LEGION_WARNINGS_FATAL: "1"

.ctest: &ctest
  TEST_CTEST: "1"

.regent: &regent
  TEST_LEGION_CXX: "0"
  TEST_REALM: "0"
  TEST_FUZZER: "0"
  NO_PRETTY: "1"
.regent_pretty: &regent_pretty
  TEST_LEGION_CXX: "0"
  TEST_REALM: "0"
  TEST_FUZZER: "0"
  REGENT_SAFE_COMPILER: "1"
.parallel: &parallel
  REGENT_JOBS: "2"
.incremental: &incremental
  REGENT_INCREMENTAL: "1"

.external1: &external1
  TEST_PY_ARGS: "--test=external1"
.external2: &external2
  TEST_PY_ARGS: "--test=external2 --test=private"

.tsan: &tsan
  SANITIZER_TYPE: "TSAN"
.ubsan: &ubsan
  SANITIZER_TYPE: "UBSAN"
  UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1"
.asan: &asan
  SANITIZER_TYPE: "ASAN"

.jupyter: &jupyter
  TEST_JUPYTER: "1"
  TEST_PY_ARGS: "--test=jupyter"

###
### Setup
###

# These commands will run before each job.
before_script:
  - set -e
  - uname -a
  - |
    if [[ "$(uname)" = "Linux" ]]; then
      export THREADS=$(nproc --all)
      if [[ "$USE_CCACHE" -eq 1 ]]; then
        apt install -y ccache
      fi
    elif [[ "$(uname)" = "Darwin" ]]; then
      export THREADS=$(sysctl -n hw.ncpu)
    else
      echo "Unknown platform. Setting THREADS to 1."
      export THREADS=1
    fi
  - |
    if [[ "$UCX_SOURCE" != "" ]]; then
      mkdir ucx
      cd ucx
      wget -O - $UCX_SOURCE | tar --strip-components=1 -zxf -
      UCX_INSTALL_ROOT=$PWD/install
      UCX_CONFIG_OPTS="--disable-numa"
      contrib/configure-release --prefix=$UCX_INSTALL_ROOT $UCX_CONFIG_OPTS
      make -j8 install
      PATH=$PATH:$UCX_INSTALL_ROOT/bin
      export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$UCX_INSTALL_ROOT/lib
      # force inter-rank communication through tcp to make it "slow"
      export UCX_TLS=tcp
      cd ..
    fi
    if [[ "$REALM_NETWORKS" != "" ]]; then
      # OpenMPI:
      # export LAUNCHER="mpirun -n 2 -x TERRA_PATH -x INCLUDE_PATH -x LD_LIBRARY_PATH -x LG_RT_DIR -x USE_RDIR"
      # MPICH:
      export LAUNCHER="mpirun -n 2"
      export MPICH_CXX=${CXX}
      if [[ "$REALM_NETWORKS" == gasnet* ]]; then
        if [[ "$EMBED_GASNET" -eq 1 ]]; then
          if [[ "$EMBED_GASNET_LOCAL" -eq 1 ]]; then
            # clone gasnet, but make it read only since we want an embedded build
            git clone https://github.com/StanfordLegion/gasnet.git gasnet
            chmod -R a-w gasnet
            export EMBED_GASNET_SRC="$PWD/gasnet"
          fi
        else
          git clone https://github.com/StanfordLegion/gasnet.git gasnet
          make -C gasnet -j$THREADS
          if [[ "$GASNET_DEBUG" -eq 1 ]]; then
            export GASNET_ROOT="$PWD/gasnet/debug"
          else
            export GASNET_ROOT="$PWD/gasnet/release"
          fi
        fi
      fi
      if [[ "$REALM_NETWORKS" == ucx ]]; then
        apt-get install -y autoconf automake m4 libtool
        git clone https://github.com/openucx/ucx ucx-git
        cd ucx-git
        git checkout 5f26dd4
        ./autogen.sh
        if [[ "$CUDA" != "" ]]; then
          ./contrib/configure-release-mt --with-cuda=$CUDA
          # CI tests are run on a single node. So disable shared-memory and cuda ipc
          # transports to be able to test some network functionality.
          export UCX_TLS=^sm,cuda_ipc
        else
          ./contrib/configure-release-mt --without-cuda
          export UCX_TLS=^sm
        fi
        make -j install
        cd -
        # build mpi bootstrap .so file.
        # the mpicc used here must be consistent with launcher's mpirun.
        mpicc -shared -fPIC -o realm_ucp_bootstrap_mpi.so runtime/realm/ucx/bootstrap/bootstrap_mpi.c
        export REALM_UCP_BOOTSTRAP_PLUGIN=$PWD/realm_ucp_bootstrap_mpi.so
      fi
    fi
  - |
    if [ "$USE_CCACHE" -eq 1 ]; then
      export CCACHE_COMPILER=${CXX}
      ccache --zero-stats
      ccache --show-stats
    fi
  - |
    if [[ "$USE_CUDA" -eq 1 ]]; then
      # make sure dynamic loading finds the right libcudart.so
      export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${CUDA}/lib64"
      export PATH=$PATH:${CUDA}/bin
      # Try to use the stubs over the driver
      export CUDA_PATH="${CUDA}/lib64:${CUDA}/lib64/stubs:/usr/local/nvidia/lib64"
    fi
  - |
    if [[ "$USE_KOKKOS" -eq 1 ]]; then
      # git clone -b $KOKKOS_BRANCH -- $KOKKOS_REPO kokkos
      # FIXME (Wei): since we are using the dev branch, pick a stable hash for now
      #   remove it once the dev branch has been merged into master
      # git -C kokkos checkout 14deae4036aa7165c55fc53bf8634f654ac6e257
      # git -C kokkos log -n 1
      mkdir kokkos
      cd kokkos
      git init
      git remote add origin $KOKKOS_REPO
      git fetch --depth 1 origin $KOKKOS_COMMIT
      git checkout FETCH_HEAD
      cd ..
      mkdir kokkos/build
      cd kokkos/build
      KOKKOS_CMAKE_OPTS="-DCMAKE_INSTALL_PREFIX=../install -DCMAKE_CXX_STANDARD=${CXX_STANDARD}"
      # the following line is a workaround for Kokkos issue #2652
      KOKKOS_CMAKE_OPTS="$KOKKOS_CMAKE_OPTS -DCMAKE_CXX_FLAGS=-DKOKKOS_IMPL_TURN_OFF_CUDA_HOST_INIT_CHECK"
      if [[ "$USE_OPENMP" -eq 1 ]]; then
        KOKKOS_CMAKE_OPTS="$KOKKOS_CMAKE_OPTS -DKokkos_ENABLE_OPENMP=ON"
      else
        KOKKOS_CMAKE_OPTS="$KOKKOS_CMAKE_OPTS -DKokkos_ENABLE_SERIAL=ON"
      fi
      if [[ "$SHARED_OBJECTS" -eq 1 ]]; then
        KOKKOS_CMAKE_OPTS="$KOKKOS_CMAKE_OPTS -DBUILD_SHARED_LIBS=ON"
      fi
      if [[ "$USE_CUDA" -eq 1 ]]; then
        KOKKOS_CMAKE_OPTS="$KOKKOS_CMAKE_OPTS -DKokkos_ENABLE_CUDA=ON -DKokkos_ENABLE_CUDA_LAMBDA=ON -DKokkos_ENABLE_CUDA_UVM=OFF"
        # Pick a base CUDA architecture to enable
        KOKKOS_CMAKE_OPTS="$KOKKOS_CMAKE_OPTS -DKokkos_ARCH_${KOKKOS_CUDA_ARCH}=ON"
        # Force Kokkos build to always use the nvcc wrapper (even for clang)
        # This allows kokkos to build SASS and PTX in a forward compatible way,
        # allowing older toolkits to work on newer GPUs in CI
        # This still requires a patch to Kokkos, coming in 4.0:
        # https://github.com/kokkos/kokkos/pull/5527
        KOKKOS_CMAKE_OPTS="$KOKKOS_CMAKE_OPTS -DCMAKE_CXX_COMPILER=$(realpath ../bin)/nvcc_wrapper"
        export NVCC_WRAPPER_DEFAULT_COMPILER="$CXX"
        #KOKKOS_CMAKE_OPTS="$KOKKOS_CMAKE_OPTS -DKokkos_CUDA_DIR=${CUDA}"
        # Make sure KOKKOS and Legion use the same compiler for NVCC as for HOST
        # code and pass this into the -ccbin argument for nvcc
        export CUDAHOSTCXX="$CXX"
        #export CUDA_PATH="${CUDA}/lib64:${CUDA}/lib64/stubs"
        #export CUDA_TOOLKIT_ROOT_DIR=$CUDA
        #export CUDA_PATH="${CUDA}/bin"

        # kokkos expects (the right) nvcc to be in the path, both for toolkit
        #  detection as well as for nvcc_wrapper (if used)
        PATH=$PATH:${CUDA}/bin
      else
        KOKKOS_CMAKE_OPTS="$KOKKOS_CMAKE_OPTS -DCMAKE_CXX_COMPILER=$CXX"
      fi
      echo cmake $KOKKOS_CMAKE_OPTS ..
      cmake $KOKKOS_CMAKE_OPTS ..
      make -j$THREADS install
      cd ../..
      # tell cmake where to find the Kokkos install
      export Kokkos_DIR=`pwd`/kokkos/install

      # now build kokkos-kernels
      # git clone -b $KKERNELS_BRANCH -- $KKERNELS_REPO kokkos-kernels
      # FIXME (Wei): since we are using the dev branch, pick a stable hash for now
      #   remove it once the dev branch has been merged into master
      # git -C kokkos-kernels checkout bce06df1aa1fc0634aecfb579eb41341db9c9bcf
      # git -C kokkos-kernels log -n 1
      mkdir kokkos-kernels
      cd kokkos-kernels
      git init
      git remote add origin $KKERNELS_REPO
      git fetch --depth 1 origin $KKERNELS_COMMIT
      git checkout FETCH_HEAD
      cd ..
      mkdir kokkos-kernels/build
      cd kokkos-kernels/build
      KKERNELS_CMAKE_OPTS="-DCMAKE_INSTALL_PREFIX=../install -DCMAKE_CXX_STANDARD=${CXX_STANDARD}"
      # FIXME (Wei): disable sparse for now because it does not compile
      KKERNELS_CMAKE_OPTS="$KKERNELS_CMAKE_OPTS -DCMAKE_CXX_COMPILER=${KOKKOS_CXX_COMPILER} -DKokkosKernels_ENABLE_ALL_COMPONENTS=OFF -DKokkosKernels_ENABLE_BLAS=ON -DKokkosKernels_ENABLE_SPARSE=OFF"
      if [[ "$SHARED_OBJECTS" -eq 1 ]]; then
        KKERNELS_CMAKE_OPTS="$KKERNELS_CMAKE_OPTS -DBUILD_SHARED_LIBS=ON"
      fi
      echo cmake $KKERNELS_CMAKE_OPTS ..
      cmake $KKERNELS_CMAKE_OPTS ..
      make -j$THREADS install
      cd ../..
      # tell cmake where to find the Kokkos install
      export KokkosKernels_DIR=`pwd`/kokkos-kernels/install

      # enable WAR for: https://github.com/kokkos/kokkos-kernels/issues/757
      export REALM_DEFAULT_ARGS="-cuda:nongpusync 0"

      if [[ "$USE_CUDA" -eq 1 ]]; then
        # There are warnings when compiling kokkos_interop.cc, so disable WARN_AS_ERROR
        export WARN_AS_ERROR=0
      fi

    fi
  - |
    if [[ "$TEST_JUPYTER" -eq 1 ]]; then
      pip3 install notebook==6.4.10
    fi
  - |
    if [[ "$USE_PROF" -eq 1 ]]; then
      wget https://bootstrap.pypa.io/get-pip.py
      python3 get-pip.py
      pip3 install mypy==0.971
    fi


###
### Tags
###

.linux: &linux
  tags:
    - linux
.macos: &macos
  tags:
    - macos

# For MSVC-in-wine tests:
.msvc_wine: &msvc_wine
  tags:
    - linux
    - msvc_wine

# For compute-heavy tests:
.linux_compute: &linux_compute
  tags:
    - linux
    - compute

# For tests that require AVX:
.linux_avx: &linux_avx
  tags:
    - linux
    - avx

# For CUDA tests, always use a specific machine:
.p100: &p100
  tags:
    - p100

.p100_base: &p100_base
  IS_P100_JOB: "1"
  USE_CUDA: "1"
  # docker puts both 32 and 64 bit libcuda.so's in LD_LIBRARY_PATH, but that confuses Kokkos's cmake
  LD_LIBRARY_PATH: "/usr/local/nvidia/lib64"

.p100_cuda102: &p100_cuda102
  <<: *p100_base
  CUDA: "/usr/local/cuda-10.2"
  CUDA_TOOLKIT_ROOT: "/usr/local/cuda-10.2"
  USE_NVTX: "1"

.p100_cuda110: &p100_cuda110
  <<: *p100_base
  CUDA: "/usr/local/cuda-11.0"
  CUDA_TOOLKIT_ROOT: "/usr/local/cuda-11.0"
  USE_NVTX: "1"

.p100_cuda112: &p100_cuda112
  <<: *p100_base
  CUDA: "/usr/local/cuda-11.2"
  CUDA_TOOLKIT_ROOT: "/usr/local/cuda-11.2"
  USE_NVTX: "1"

###
### Docker Image
###

# Each job will run in a fresh container with this image.
.image_2004: &image_2004
  image: registry.gitlab.com/stanfordlegion/legion/gitlab-ci-20.04:230203
.image_2204: &image_2204
  image: registry.gitlab.com/stanfordlegion/legion/gitlab-ci-22.04:230203
# FIXME (Elliott): I can't get this job to work in the new 22.04 image, so just leave the old image for now
.image_kokkos_openmp: &image_kokkos_openmp
  image: registry.gitlab.com/stanfordlegion/legion/gitlab-ci-20.04:210923
.image_hpcsdk: &image_hpcsdk
  image: registry.gitlab.com/stanfordlegion/legion/gitlab-ci-hpcsdk:221201
.image_icc: &image_icc
  image: registry.gitlab.com/stanfordlegion/legion/gitlab-ci-icc
.image_msvc169: &image_msvc169
  image: registry.gitlab.com/stanfordlegion/legion/gitlab-ci-msvc

###
### Tests
###

# rules for which jobs will run
.test_rules: &test_rules
  rules:
    # some jobs are tagged and can only run on certain hardware - a CI variable
    #  is used to enable these jobs in case that hardware is offline
    - if: '$IS_P100_JOB == "1" && $RUN_P100_JOBS != "1"'
      when: never
    # some scheduled runs are only looking at specific subsets of functionality
    - if: '$ONLY_KOKKOS_JOBS == "1" && $USE_KOKKOS != "1"'
      when: never
    # do not automatically run pipelines for pull requests
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: never
    # commits only run a short suite of jobs
    - if: '$CI_PIPELINE_SOURCE == "push" && $IS_SHORT_JOB != "1"'
      when: never
    # if none of the above exclusions apply, run the job
    - when: always

.short: &short
  IS_SHORT_JOB: "1"

# Each job will run this set of tests.
.tests: &tests
  <<: *test_rules
  script:
    - ./tools/add_github_host_key.sh
    - grep 'model name' /proc/cpuinfo | uniq -c || true
    - which $CXX
    - $CXX --version
    - |
      if [[ "$USE_DEFCHECK" -eq 1 ]]; then
        export DEFCHECK_CXX="${CXX}"
        export CXX=`pwd`/tools/cxx_defcheck
        # if we're using MPI, make sure mpicxx uses the right compiler too
        export OMPI_CXX=`pwd`/tools/cxx_defcheck
        export MPICH_CXX=`pwd`/tools/cxx_defcheck
      fi
    - free
    - |
      if [[ -z "$TEST_PYTHON_EXE" ]]; then
        export TEST_PYTHON_EXE=`which python3 python | head -1`
      fi
    - |
      if [[ "$USE_CCACHE" -eq 1 ]]; then
          export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_CXX_COMPILER_LAUNCHER=ccache"
      fi
    - |
      if [[ -n "$CXX_STANDARD" ]]; then
        export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_CXX_STANDARD=${CXX_STANDARD}"
      fi
    - $TEST_PYTHON_EXE ./test.py --tmp-dir ${CI_PROJECT_DIR}/tmp-build
    - |
      if [[ "$USE_CCACHE" -eq 1 ]]; then
        ccache --show-stats
      fi
  retry:
    max: 0
    when:
      - runner_system_failure
      - unknown_failure

# Some additional tests are only run on certain configurations.
.external_tests: &external_tests
  <<: *test_rules
  script:
    - ./tools/add_github_host_key.sh
    - |
      if [[ -n $GITLAB_DEPLOY_KEY ]]; then
        eval $(ssh-agent -s)
        ssh-add <(echo "$GITLAB_DEPLOY_KEY")
      fi
    - unset USE_DEFCHECK
    - |
      if [[ -z "$TEST_PYTHON_EXE" ]]; then
        export TEST_PYTHON_EXE=`which python3 python | head -1`
      fi
    - $TEST_PYTHON_EXE ./test.py $TEST_PY_ARGS --tmp-dir ${CI_PROJECT_DIR}/tmp
  retry:
    max: 0
    when:
      - runner_system_failure
      - unknown_failure

# testing MSVC builds is kinda hacky right now - not supported by test.py
.msvc_tests: &msvc_tests
  <<: *test_rules
  script:
    - pwd
    - ln -s `pwd` /home/wine/.wine/drive_c/repo
    - cd /home/wine/.wine/drive_c
    - su wine -c "wine64 cmd /c 'x64.bat && cmake -G Ninja -S repo -B build -DLegion_BUILD_REALM_ONLY=ON -DLegion_BUILD_REALM_TESTS=ON -DLegion_ENABLE_TESTING=ON'"
    - "# wine doesn't seem to cope with concurrent file access, so use a serial build"
    - su wine -c "wine64 cmd /c 'x64.bat && cmake --build build -- -j1'"
    - su wine -c "wine64 cmd /c 'x64.bat && set PATH=C:\\Tools\\VS2019\\VC\\Redist\\MSVC\\14.28.29910\\debug_nonredist\\x64\\Microsoft.VC142.DebugCRT;C:\\Tools\\SDK\\10\\bin\\10.0.18362.0\\x64\\ucrt;%PATH% && cd build && ctest --output-on-failure'"
    - echo OK

# use a custom script for running Realm tests with sanitizers
.realm_sanitizer_tests: &realm_sanitizer_tests
  <<: *test_rules
  script:
    - mkdir build
    - cd build
    - cmake -DCMAKE_BUILD_TYPE=${SANITIZER_TYPE} -DLegion_BUILD_REALM_ONLY=ON -DLegion_BUILD_REALM_TESTS=ON -DLegion_ENABLE_TESTING=ON ..
    - make -j8
    - ctest --output-on-failure .
    - echo OK

# For performance tests, run these commands:
.perf_tests: &perf_tests
  script:
    - ./tools/add_github_host_key.sh
    - |
      if [[ -n $GITLAB_DEPLOY_KEY ]]; then
        eval $(ssh-agent -s)
        ssh-add <(echo "$GITLAB_DEPLOY_KEY")
      fi
    - git config --global user.name "Legion Testing Automation"
    - git config --global user.email "legion-testing@googlegroups.com"
    - export PERF_MIN_NODES=1
    - export PERF_MAX_NODES=1
    - $TEST_PYTHON_EXE ./test.py --test=perf --tmp-dir ${CI_PROJECT_DIR}/tmp
  retry:
    max: 0
    when:
      - runner_system_failure
      - unknown_failure

###
### Jobs
###

# Each item below defines a job.

# There are two tiers of tests.
#  1. Full test suite (small number of compilers).
#  2. Minimal test suite (other compilers).

# Run the full test suite on GCC 9 and Clang 12.

# Linux with GCC 9
# * Basic configurations
#   * Release
gcc9_cxx11_release_legion_fortran:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *release, *cxx11_normal, *legion, *fortran]
gcc9_cxx11_release_llvm_cmake_regent_pretty:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *release, *cxx11_normal, *llvm, *cmake, *regent_pretty, *incremental, *short, *shared]
#   * Debug (Privilege and Bounds Checks)
gcc9_cxx11_debug_checks_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *checks, *legion]
#   * Spy
gcc9_cxx11_debug_spy_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *spy, *legion]
gcc9_cxx11_debug_spy_python3_regent:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *spy, *python3, *regent, *parallel]
#   * Prof
gcc9_cxx11_release_prof_regent:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *release, *cxx11_normal, *prof, *regent, *parallel]
#   * C++11
gcc9_cxx11_release_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *release, *cxx11_normal, *legion]
#   * higher dimension support (5-D)
gcc9_cxx11_release_maxdim5_regent:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *release, *cxx11_normal, *maxdim5, *regent, *parallel]
gcc9_cxx11_release_maxdim5_regent_pretty:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *release, *cxx11_normal, *maxdim5, *regent_pretty, *parallel]
# * Features: one test per feature
#   * OpenMP
gcc9_cxx11_debug_openmp_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *openmp, *legion]
gcc9_cxx11_debug_openmp_regent:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *openmp, *regent]
#   * Python
gcc9_cxx11_debug_python3_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *python3, *legion, *prof, *jupyter, *short]
#   * LLVM
gcc9_cxx11_debug_llvm_cmake_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *llvm, *cmake, *legion, *ctest, *shared]
gcc9_cxx11_debug_llvm_regent:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *llvm, *regent]
#   * HDF5
gcc9_cxx11_debug_hdf5_cmake_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *hdf5, *cmake, *legion, *ctest]
gcc9_cxx11_debug_hdf5_regent_pretty:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *hdf5, *regent_pretty]
#   * GASNet-1 (legacy API)
gcc9_cxx11_release_gasnet1_mpi_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *release, *cxx11_normal, *gasnet1_mpi, *legion]
gcc9_cxx11_debug_gasnet1_mpi_cmake_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *gasnet1_mpi, *cmake, *legion, *ctest, *shared]
gcc9_cxx11_debug_gasnet1_mpi_regent:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *gasnet1_mpi, *regent, *incremental]
#   * Embedded GASNet builds (local and remote, static and shared)
gcc9_cxx11_release_gasnet1_mpi_embedlocal_cmake_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *release, *cxx11_normal, *gasnet1_mpi, *gasnet_embed_local, *cmake, *ctest, *legion, *shared]
gcc9_cxx11_release_gasnet1_mpi_embedremote_cmake_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *release, *cxx11_normal, *gasnet1_mpi, *gasnet_embed_remote, *cmake, *ctest, *legion, *shared]
gcc9_cxx11_release_shared_gasnet1_mpi_embedlocal_cmake_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *release, *shared, *cxx11_normal, *gasnet1_mpi, *gasnet_embed_local, *cmake, *ctest, *legion, *shared]
gcc9_cxx11_release_shared_gasnet1_mpi_embedremote_cmake_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *release, *shared, *cxx11_normal, *gasnet1_mpi, *gasnet_embed_remote, *cmake, *ctest, *legion, *shared]
#   * GASNet-EX (native API, 2020.11.0+)
gcc9_cxx11_release_gasnetex_mpi_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *release, *cxx11_normal, *gasnetex_mpi, *legion]
gcc9_cxx11_debug_gasnetex_mpi_cmake_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *gasnetex_mpi, *cmake, *legion, *ctest, *short, *shared]
gcc9_cxx11_debug_gasnetex_mpi_regent:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *gasnetex_mpi, *regent, *incremental, *short]
gcc9_cxx11_debug_gasnetex_mpi_debug_regent:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *gasnetex_mpi_debug, *regent, *incremental]
gcc9_cxx11_debug_gasnetex_mpi_nightly_regent:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *gasnetex_mpi_nightly, *regent, *incremental]
#   * GASNet (UCX conduit)
gcc9_cxx11_release_gasnet1_ucx_legion:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *release, *cxx11_normal, *gasnet1_ucx, *legion]
gcc9_cxx11_release_gasnet1_ucx_cmake_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *release, *cxx11_normal, *gasnet1_ucx, *cmake, *legion, *ctest, *shared]
gcc9_cxx11_release_gasnet1_ucx_regent:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *release, *cxx11_normal, *gasnet1_ucx, *regent, *incremental]
gcc9_cxx11_release_gasnetex_ucx_legion:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *release, *cxx11_normal, *gasnetex_ucx, *legion]
gcc9_cxx11_debug_gasnetex_ucx_cmake_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *gasnetex_ucx, *cmake, *legion, *ctest, *shared]
gcc9_cxx11_release_gasnetex_ucx_regent:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *release, *cxx11_normal, *gasnetex_ucx, *regent, *incremental,  *short]
#   * MPI and multi-node legion prof
gcc9_cxx11_release_mpi_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *release, *cxx11_normal, *mpi, *legion]
gcc9_cxx11_debug_mpi_cmake_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *mpi, *cmake, *legion, *ctest, *shared]
gcc9_cxx11_release_mpi_prof_regent:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *release, *cxx11_normal, *mpi, *regent, *incremental, *prof, *short]
#   * UCX
gcc9_cxx11_release_ucx_legion:
 <<: [*linux, *image_2004, *tests]
 variables:
   <<: [*gcc9, *terra12, *release, *cxx11_normal, *ucx, *legion]
gcc9_cxx11_debug_ucx_cmake_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *ucx, *cmake, *legion, *ctest, *short]
gcc9_cxx11_debug_ucx_regent:
 <<: [*linux_compute, *image_2004, *tests]
 variables:
   <<: [*gcc9, *terra12, *debug, *cxx11_normal, *ucx, *regent, *incremental]
# * Integration: Python + Regent
gcc9_cxx11_debug_python3_regent:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *python3, *regent]
# * Integration: LLVM + GASNet
gcc9_cxx11_debug_llvm_gasnetex_mpi_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *llvm, *gasnetex_mpi, *legion]
gcc9_cxx11_debug_llvm_gasnetex_mpi_regent:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *llvm, *gasnetex_mpi, *regent, *incremental]
# * Integration: LLVM + MPI
gcc9_cxx11_debug_llvm_mpi_legion:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *llvm, *mpi, *legion]
gcc9_cxx11_debug_llvm_mpi_regent:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *llvm, *mpi, *regent, *incremental]
# * Integration: LLVM + UCX
#gcc9_cxx11_debug_llvm_ucx_legion:
#  <<: [*linux_compute, *image_2004, *tests]
#  variables:
#    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *llvm, *ucx, *legion]
#gcc9_cxx11_debug_llvm_ucx_regent:
#  <<: [*linux_compute, *image_2004, *tests]
#  variables:
#    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *llvm, *ucx, *regent, *incremental]
# * Integration: HDF5 + GASNet
gcc9_cxx11_debug_hdf5_gasnetex_mpi_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *hdf5, *gasnetex_mpi, *legion]
gcc9_cxx11_debug_hdf5_gasnetex_mpi_regent:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *hdf5, *gasnetex_mpi, *regent]
# * Integration: HDF5 + MPI
gcc9_cxx11_debug_hdf5_mpi_legion:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *hdf5, *mpi, *legion]
gcc9_cxx11_debug_hdf5_mpi_regent:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *hdf5, *mpi, *regent]
# * Integration: HDF5 + UCX
#gcc9_cxx11_debug_hdf5_ucx_legion:
#  <<: [*linux_compute, *image_2004, *tests]
#  variables:
#    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *hdf5, *ucx, *legion]
#gcc9_cxx11_debug_hdf5_ucx_regent:
#  <<: [*linux, *image_2004, *tests]
#  variables:
#    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *hdf5, *ucx, *regent]
# Multi-node Legion Spy
gcc9_cxx11_debug_spy_gasnet1_mpi_regent:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *cxx11_normal, *spy, *gasnet1_mpi, *regent, *short]
gcc9_cxx11_debug_spy_mpi_regent:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *cxx11_normal, *spy, *mpi, *regent]
#gcc9_cxx11_debug_spy_ucx_regent:
#  <<: [*linux_compute, *image_2004, *tests]
#  variables:
#    <<: [*gcc9, *terra12, *cxx11_normal, *spy, *ucx, *regent]
# * Different architectures
gcc9_cxx11_32bit_debug_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_32bit_normal, *legion]
gcc9_cxx11_32bit_release_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *release, *cxx11_32bit_normal, *legion]
# * External tests
gcc9_cxx11_debug_external1:
  <<: [*linux_compute, *image_2004, *external_tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *openmp, *hdf5, *external1, *short]
gcc9_cxx11_debug_maxdim4_external2:
  <<: [*linux_compute, *image_2004, *external_tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx11_normal, *maxdim4, *openmp, *hdf5, *external2, *short]

# Linux with Clang 12
# * Basic configurations
#   * Release
clang12_cxx11_release_legion_fortran:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *release, *cxx11_normal, *legion, *fortran]
clang12_cxx11_release_llvm_cmake_regent:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *release, *cxx11_normal, *llvm, *cmake, *regent, *incremental]
#   * Debug (Privilege and Bounds Checks)
clang12_cxx11_debug_checks_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx11_normal, *checks, *legion]
#   * Spy
clang12_cxx11_debug_spy_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx11_normal, *spy, *legion]
clang12_cxx11_debug_spy_python3_regent:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx11_normal, *spy, *python3, *regent, *parallel, *short]
#   * Prof
clang12_cxx11_release_prof_regent:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *release, *cxx11_normal, *prof, *regent, *parallel, *short]
#   * C++11
clang12_cxx11_release_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *release, *cxx11_normal, *legion]
# * Features: one test per feature
#   * OpenMP
clang12_cxx11_debug_openmp_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx11_normal, *openmp, *legion]
clang12_cxx11_debug_openmp_regent:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx11_normal, *openmp, *regent]
#   * Python
clang12_cxx11_debug_python3_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx11_normal, *python3, *legion]
#   * LLVM
clang12_cxx11_debug_llvm_cmake_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx11_normal, *llvm, *cmake, *legion, *ctest]
clang12_cxx11_debug_llvm_regent:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx11_normal, *llvm, *regent]
#   * HDF5
clang12_cxx11_debug_hdf5_cmake_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx11_normal, *hdf5, *cmake, *legion, *ctest]
clang12_cxx11_debug_hdf5_regent:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx11_normal, *hdf5, *regent]
#   * GASNet
clang12_cxx11_debug_gasnetex_mpi_cmake_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx11_normal, *gasnetex_mpi, *cmake, *legion, *ctest]
clang12_cxx11_debug_gasnetex_mpi_regent:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx11_normal, *gasnetex_mpi, *regent, *incremental]
#   * MPI
clang12_cxx11_debug_mpi_cmake_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx11_normal, *mpi, *cmake, *legion, *ctest]
clang12_cxx11_debug_mpi_regent:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx11_normal, *mpi, *regent, *incremental]
#   * UCX
clang12_cxx11_debug_ucx_cmake_legion:
 <<: [*linux, *image_2004, *tests]
 variables:
   <<: [*clang12, *terra12, *debug, *cxx11_normal, *ucx, *cmake, *legion, *ctest]
clang12_cxx11_debug_ucx_regent:
 <<: [*linux_compute, *image_2004, *tests]
 variables:
   <<: [*clang12, *terra12, *debug, *cxx11_normal, *ucx, *regent, *incremental]
# * Integration: Python + Regent
clang12_cxx11_debug_python3_regent:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx11_normal, *python3, *regent]
# * Integration: LLVM + GASNet
clang12_cxx11_debug_llvm_gasnetex_mpi_legion:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx11_normal, *llvm, *gasnetex_mpi, *legion]
clang12_cxx11_debug_llvm_gasnetex_mpi_regent:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx11_normal, *llvm, *gasnetex_mpi, *regent, *incremental]
# * Integration: LLVM + MPI
clang12_cxx11_debug_llvm_mpi_legion:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx11_normal, *llvm, *mpi, *legion]
clang12_cxx11_debug_llvm_mpi_regent:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx11_normal, *llvm, *mpi, *regent, *incremental]
# * Integration: LLVM + UCX
#clang12_cxx11_debug_llvm_ucx_legion:
#  <<: [*linux_compute, *image_2004, *tests]
#  variables:
#    <<: [*clang12, *terra12, *debug, *cxx11_normal, *llvm, *ucx, *legion]
#clang12_cxx11_debug_llvm_ucx_regent:
#  <<: [*linux_compute, *image_2004, *tests]
#  variables:
#    <<: [*clang12, *terra12, *debug, *cxx11_normal, *llvm, *ucx, *regent, *incremental]
# * Integration: HDF5 + GASNet
clang12_cxx11_debug_hdf5_gasnetex_mpi_legion:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx11_normal, *hdf5, *gasnetex_mpi, *legion]
clang12_cxx11_debug_hdf5_gasnetex_mpi_regent:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx11_normal, *hdf5, *gasnetex_mpi, *regent, *incremental]
# * Integration: HDF5 + MPI
clang12_cxx11_debug_hdf5_mpi_legion:
  <<: [*linux_compute, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx11_normal, *hdf5, *mpi, *legion]
clang12_cxx11_debug_hdf5_mpi_regent:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx11_normal, *hdf5, *mpi, *regent, *incremental]
# * Integration: HDF5 + UCX
#clang12_cxx11_debug_hdf5_ucx_legion:
#  <<: [*linux_compute, *image_2004, *tests]
#  variables:
#    <<: [*clang12, *terra12, *debug, *cxx11_normal, *hdf5, *ucx, *legion]
#clang12_cxx11_debug_hdf5_ucx_regent:
#  <<: [*linux, *image_2004, *tests]
#  variables:
#    <<: [*clang12, *terra12, *debug, *cxx11_normal, *hdf5, *ucx, *regent, *incremental]
# * Integration: HDF5 + GASNet
# clang12_cxx11_debug_spy_gasnet_regent:
#   <<: [*linux_compute, *image_2004, *tests]
#   variables:
#     <<: [*clang12, *terra12, *debug, *cxx11_normal, *spy, *gasnet, *regent]
# * Different architectures
clang12_cxx11_32bit_debug_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx11_32bit_normal, *legion]
clang12_cxx11_32bit_release_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *release, *cxx11_32bit_normal, *legion, *short]
# * External tests
clang12_cxx11_debug_external1:
  <<: [*linux_compute, *image_2004, *external_tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx11_normal, *openmp, *hdf5, *external1]
clang12_cxx11_debug_maxdim4_external2:
  <<: [*linux_compute, *image_2004, *external_tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx11_normal, *maxdim4, *openmp, *hdf5, *external2]

# # macOS with Clang 3.5
# macos_clang12_cxx11_debug:
#   <<: [*macos, *tests]
#   variables:
#     <<: [*clang12, *debug, *cxx11_normal]

# Run minimal tests on other compilers.

# GCC 9 with extra hardening
gcc9_cxx11_hardened_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *release, *cxx11_hardened, *legion]
gcc9_cxx11_hardened_regent:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *release, *cxx11_hardened, *regent, *parallel]

# GCC 9 with C++ 14
gcc9_cxx14_debug_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx14_normal, *legion]
gcc9_cxx14_release_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *release, *cxx14_normal, *legion]
gcc9_cxx14_debug_regent:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *debug, *cxx14_normal, *regent]
gcc9_cxx14_release_regent_pretty:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc9, *terra12, *release, *cxx14_normal, *regent_pretty]

# GCC 10 with C++ 17
gcc10_cxx17_debug_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc10, *terra12, *debug, *cxx17_normal, *legion]
gcc10_cxx17_release_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc10, *terra12, *release, *cxx17_normal, *legion]
gcc10_cxx17_debug_regent:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc10, *terra12, *debug, *cxx17_normal, *regent]
gcc10_cxx17_release_regent_pretty:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc10, *terra12, *release, *cxx17_normal, *regent_pretty]

# GCC 11 with C++ 20
gcc11_cxx20_debug_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc11, *terra12, *debug, *cxx20_normal, *legion]
gcc11_cxx20_release_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc11, *terra12, *release, *cxx20_normal, *legion]
gcc11_cxx20_debug_regent:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc11, *terra12, *debug, *cxx20_normal, *regent]
gcc11_cxx20_release_regent_pretty:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*gcc11, *terra12, *release, *cxx20_normal, *regent_pretty]

# GCC 12 with C++ 20
gcc12_cxx20_debug_legion:
  <<: [*linux, *image_2204, *tests]
  variables:
    <<: [*gcc12, *terra14, *debug, *cxx20_normal, *legion]
gcc12_cxx20_release_legion:
  <<: [*linux, *image_2204, *tests]
  variables:
    <<: [*gcc12, *terra14, *release, *cxx20_normal, *legion]
gcc12_cxx20_debug_regent:
  <<: [*linux, *image_2204, *tests]
  variables:
    <<: [*gcc12, *terra14, *debug, *cxx20_normal, *regent]
gcc12_cxx20_release_regent_pretty:
  <<: [*linux, *image_2204, *tests]
  variables:
    <<: [*gcc12, *terra14, *release, *cxx20_normal, *regent_pretty]

# Clang 14 with C++ 17/20
clang14_cxx17_debug_legion:
  <<: [*linux, *image_2204, *tests]
  variables:
    <<: [*clang14, *terra14, *debug, *cxx17_normal, *legion]
clang14_cxx17_release_legion:
  <<: [*linux, *image_2204, *tests]
  variables:
    <<: [*clang14, *terra14, *release, *cxx17_normal, *legion]
clang14_cxx17_debug_regent:
  <<: [*linux, *image_2204, *tests]
  variables:
    <<: [*clang14, *terra14, *debug, *cxx17_normal, *regent]
clang14_cxx17_release_regent:
  <<: [*linux, *image_2204, *tests]
  variables:
    <<: [*clang14, *terra14, *release, *cxx17_normal, *regent]
clang14_cxx20_debug_regent:
  <<: [*linux, *image_2204, *tests]
  variables:
    <<: [*clang14, *terra14, *debug, *cxx20_normal, *regent]

# CUDA tests:
p100_cuda102_gcc8_cxx11_release_cuda_legion:
  <<: [*p100, *tests]
  variables:
    <<: [*gcc8, *release, *cxx11_normal, *p100_cuda102, *legion, *cmake, *ctest, *shared]
p100_cuda102_gcc8_cxx11_debug_checks_cuda_legion:
  <<: [*p100, *tests]
  variables:
    <<: [*gcc8, *debug, *cxx11_normal, *checks, *p100_cuda102, *legion, *short, *cmake, *ctest, *shared]
p100_cuda102_gcc8_cxx11_release_cmake_cuda_legion:
  <<: [*p100, *tests]
  variables:
    <<: [*gcc8, *release, *cxx11_normal, *p100_cuda102, *cmake, *legion, *ctest, *shared]
p100_cuda110_gcc9_cxx14_debug_cmake_cuda_dynamic_legion:
  <<: [*p100, *tests]
  variables:
    <<: [*gcc9, *debug, *cxx14_normal, *p100_cuda110, *cmake, *legion, *cuda_dynamic, *ctest, *short, *shared]
p100_cuda110_gcc9_cxx14_release_cuda_legion:
  <<: [*p100, *tests]
  variables:
    <<: [*gcc9, *release, *cxx14_normal, *p100_cuda110, *legion, *cmake, *ctest, *shared]
p100_cuda112_gcc9_cxx17_debug_cuda_legion:
  <<: [*p100, *tests]
  variables:
    <<: [*gcc9, *debug, *cxx14_normal, *p100_cuda112, *legion, *short, *cmake, *ctest, *shared]
p100_cuda112_gcc9_cxx17_release_cmake_cuda_legion:
  <<: [*p100, *tests]
  variables:
    <<: [*gcc9, *release, *cxx17_normal, *p100_cuda112, *cmake, *legion, *ctest, *shared]
p100_cuda102_clang7_cxx11_release_cuda_regent:
  <<: [*p100, *tests]
  variables:
    <<: [*clang7, *llvm7, *release, *cxx11_normal, *p100_cuda102, *regent, *cmake, *ctest, *short]
p100_cuda102_clang7_cxx11_release_cmake_cuda_regent:
  <<: [*p100, *tests]
  variables:
    <<: [*clang7, *llvm7, *release, *cxx11_normal, *p100_cuda102, *cmake, *ctest, *regent, *shared]

# CUDA+GASNet tests:
p100_cuda112_gcc9_cxx14_debug_gasnetex_mpi_cmake_cuda_legion:
  <<: [*p100, *tests]
  variables:
    <<: [*gcc9, *debug, *cxx14_normal, *p100_cuda112, *cmake, *gasnetex_mpi, *legion, *ctest, *short]
p100_cuda112_gcc9_cxx14_release_gasnetex_mpi_cuda_regent:
  <<: [*p100, *tests]
  variables:
    <<: [*gcc9, *llvm9, *release, *cxx14_normal, *p100_cuda112, *gasnetex_mpi, *regent]

# CUDA+UCX tests:
p100_cuda112_gcc9_cxx14_debug_ucx_cmake_cuda_legion:
 <<: [*p100, *tests]
 variables:
   <<: [*gcc9, *debug, *cxx14_normal, *p100_cuda112, *cmake, *ucx, *legion, *ctest, *short]
p100_cuda112_gcc9_cxx14_release_ucx_cuda_regent:
  <<: [*p100, *tests]
  variables:
    <<: [*gcc9, *llvm9, *release, *cxx14_normal, *p100_cuda112, *ucx, *regent]

# Kokkos tests:
clang12_cxx17_debug_kokkos_legion:
  <<: [*linux, *image_2004, *tests]
  variables:
    <<: [*clang12, *terra12, *debug, *cxx17_normal, *kokkos, *cmake, *legion, *ctest]
# FIXME (Elliott): I can't get this job to work in the new 22.04 image, so just leave the old image for now
clang10_cxx17_debug_kokkos_openmp_legion:
  <<: [*linux, *image_kokkos_openmp, *tests]
  variables:
    <<: [*clang10, *debug, *cxx17_normal, *kokkos, *openmp, *cmake, *legion, *ctest]
# clang14_cxx17_debug_kokkos_openmp_legion:
#   <<: [*linux, *image_2204, *tests]
#   variables:
#     <<: [*clang14, *terra14, *debug, *cxx17_normal, *kokkos, *openmp, *cmake, *legion, *ctest]
p100_cuda112_clang10_cxx17_debug_kokkos_cuda_legion:
  <<: [*p100, *tests]
  variables:
    <<: [*clang10, *debug, *cxx17_normal, *p100_cuda112, *kokkos, *cmake, *legion, *ctest]
p100_cuda112_clang10_cxx17_debug_kokkos_cuda_openmp_legion:
  <<: [*p100, *tests]
  variables:
    <<: [*clang10, *debug, *cxx17_normal, *p100_cuda112, *kokkos, *openmp, *cmake, *legion, *ctest, *short]
p100_cuda112_gcc9_cxx17_debug_kokkos_cuda_openmp_legion:
  <<: [*p100, *tests]
  variables:
    <<: [*gcc9, *debug, *cxx17_normal, *p100_cuda112, *kokkos, *openmp, *cmake, *legion, *ctest]
p100_cuda112_gcc9_cxx17_debug_shared_kokkos_cuda_openmp_legion:
  <<: [*p100, *tests]
  variables:
    <<: [*gcc9, *debug, *shared, *cxx17_normal, *p100_cuda112, *kokkos, *openmp, *cmake, *legion, *ctest, *short]

# HPCSDK (nvc++) tests:
# Make all HPC SDK tests use cmake since nvc++ is really slow and can't handle doing compile+link
# for all tests without exceeding the two hour timeout, this means we're missing coverage of building
# with make for nvc++ but we'll just have to live with that
hpcsdk2211_cxx11_release_cmake_legion:
  <<: [*linux_avx, *image_hpcsdk, *tests]
  variables:
    <<: [*hpcsdk2211, *release, *cxx11_normal, *cmake, *legion, *ctest]
hpcsdk2211_cxx11_debug_cmake_legion:
  <<: [*linux_avx, *image_hpcsdk, *tests]
  variables:
    <<: [*hpcsdk2211, *debug, *cxx11_normal, *cmake, *legion, *ctest, *short]

# Intel compiler (icc/icpc) tests:
icc212_cxx11_release_legion:
  <<: [*linux, *image_icc, *tests]
  variables:
    <<: [*icc212, *release, *cxx11_normal, *legion]
icc212_cxx11_debug_cmake_legion:
  <<: [*linux, *image_icc, *tests]
  variables:
    <<: [*icc212, *debug, *cxx11_normal, *cmake, *legion, *ctest]

# MSVC (Realm only):
msvc169_realm:
  <<: [*msvc_wine, *image_msvc169, *msvc_tests]
  variables:
    <<: [*short]

# sanitizer tests (Realm only):
gcc9_tsan_realm:
  <<: [*linux, *image_2004, *realm_sanitizer_tests]
  variables:
    <<: [*gcc9, *cxx11_normal, *tsan, *short]
gcc9_ubsan_realm:
  <<: [*linux, *image_2004, *realm_sanitizer_tests]
  variables:
    <<: [*gcc9, *cxx11_normal, *ubsan, *short]
gcc9_asan_realm:
  <<: [*linux, *image_2004, *realm_sanitizer_tests]
  variables:
    <<: [*gcc9, *cxx11_normal, *asan, *short]
