spack:

  view: false

  concretizer:
    reuse: False
    unify: False

  modules:
    default:
      'enable:': [lmod]
      lmod:
        defaults: [cray-mpich@8.1.17]
        core_compilers: [cce@14.0.2]
        verbose: true
        hash_length: 0
        hierarchy: [mpi]
        projections: {}
        core_specs: []
        all:
          autoload: direct
          environment:
            set:
              ${PACKAGE}_ROOT: ${PREFIX}
          suffixes:
            +rocm: rocm
        py-warpx:
          suffixes:
            ^warpx dims=2: dims2
            ^warpx dims=3: dims3
            ^warpx dims=rz: dimsRZ
        exclude_implicits: true
        include: [cray-mpich, autoconf, automake, cmake]
    prefix_inspections:
      lib64: [LD_LIBRARY_PATH]
      lib: [LD_LIBRARY_PATH]

  compilers:
  - compiler:
      spec: cce@14.0.2
      paths:
        cc: cc
        cxx: CC
        f77: ftn
        fc: ftn
      flags: {}
      operating_system: sles15
      target: any
      modules:
      - PrgEnv-cray/8.3.3
      - cce/14.0.2
      - xpmem
      - craype-x86-trento
      - libfabric
      - cray-pmi/6.1.3
      environment:
        set:
          RFE_811452_DISABLE: '1'
        append_path:
          PKG_CONFIG_PATH: /opt/cray/xpmem/2.4.4-2.3_11.2__gff0e1d9.shasta/lib64/pkgconfig
          LD_LIBRARY_PATH: /opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
        prepend_path:
          LD_LIBRARY_PATH: /opt/cray/pe/pmi/6.1.3/lib
      extra_rpaths:
      - /opt/cray/pe/gcc-libs
      - /opt/cray/gcc-libs
  - compiler:
      spec: gcc@11.2.0
      paths:
        cc: gcc
        cxx: g++
        f77: gfortran
        fc: gfortran
      flags: {}
      operating_system: sles15
      target: any
      modules:
      - PrgEnv-gnu
      - gcc/11.2.0
      environment:
        set:
          RFE_811452_DISABLE: '1'
        append_path:
          LD_LIBRARY_PATH: /opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
        prepend_path:
          LD_LIBRARY_PATH: /opt/cray/pe/pmi/6.1.3/lib
      extra_rpaths:
      - /opt/cray/pe/gcc-libs
      - /opt/cray/gcc-libs

  packages:
    all:
      require: "%cce@14.0.2"
      compiler: [cce@14.0.2]
      providers:
        blas: [openblas]
        lapack: [openblas]
        mpi: [cray-mpich]
        tbb: [intel-tbb]
        scalapack: [netlib-scalapack]
      target: [zen3]
      variants: +mpi
    binutils:
      variants: +ld +gold +headers +libiberty ~nls
    hdf5:
      variants: +fortran +hl +shared
    libfabric:
      variants: fabrics=sockets,tcp,udp,rxm
    libunwind:
      variants: +pic +xz
    ncurses:
      require: "@6.3 +termlib"
    openblas:
      require: "@0.3.20"
      variants: threads=openmp
    python:
      require: "@3.7.15"
    trilinos:
      variants: +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext
        +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu
        +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos
        +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
    xz:
      variants: +pic
    mesa:
      version: [21.3.8]

    unzip:
      require: "%gcc"
    dyninst:
      require: "%gcc"
    tbb:
      require: "intel-tbb"
    elfutils:
      variants: +bzip2 ~nls +xz
      require: "%gcc"
    py-numpy:
      require: "@1.21.6"
    paraview:
      require: ~qt +osmesa
    visit:
      require: ~gui +osmesa

    # FACILITY REQUESTED PREFERENCES
    petsc:
      variants: +batch
    parallel-netcdf:
      variants: +cxx +fortran +fpic
    netlib-scalapack:
      variants: +fpic
    netcdf-c:
      variants: ~hdf4 +mpi +parallel-netcdf +shared
    fftw:
      variants: precision=float,double,long_double
    darshan-runtime:
      variants: scheduler=slurm

    # EXTERNALS
    cray-mpich:
      buildable: false
      externals:
      - spec: cray-mpich@8.1.17 %cce@14.0.2
        prefix: /opt/cray/pe/mpich/8.1.17/ofi/crayclang/10.0
        modules:
        - cray-mpich/8.1.17
        - libfabric/1.15.0.0
    libfabric:
      buildable: false
      externals:
      - spec: libfabric@1.15
        modules:
        - libfabric/1.15.0.0

    # ROCm 5.4.0
    comgr:
      buildable: false
      externals:
      - spec: comgr@5.4.0
        prefix: /opt/rocm-5.4.0/
        modules: [rocm/5.4.0]
    hip-rocclr:
      buildable: false
      externals:
      - spec: hip-rocclr@5.4.0
        prefix: /opt/rocm-5.4.0/hip
        modules: [rocm/5.4.0]
    hipblas:
      buildable: false
      externals:
      - spec: hipblas@5.4.0
        prefix: /opt/rocm-5.4.0/
        modules: [rocm/5.4.0]
    hipcub:
      buildable: false
      externals:
      - spec: hipcub@5.4.0
        prefix: /opt/rocm-5.4.0/
        modules: [rocm/5.4.0]
    hipfft:
      buildable: false
      externals:
      - spec: hipfft@5.4.0
        prefix: /opt/rocm-5.4.0/
        modules: [rocm/5.4.0]
    hipsparse:
      buildable: false
      externals:
      - spec: hipsparse@5.4.0
        prefix: /opt/rocm-5.4.0/
        modules: [rocm/5.4.0]
    miopen-hip:
      buildable: false
      externals:
      - spec: hip-rocclr@5.4.0
        prefix: /opt/rocm-5.4.0/
        modules: [rocm/5.4.0]
    miopengemm:
      buildable: false
      externals:
      - spec: miopengemm@5.4.0
        prefix: /opt/rocm-5.4.0/
        modules: [rocm/5.4.0]
    rccl:
      buildable: false
      externals:
      - spec: rccl@5.4.0
        prefix: /opt/rocm-5.4.0/
        modules: [rocm/5.4.0]
    rocblas:
      buildable: false
      externals:
      - spec: rocblas@5.4.0
        prefix: /opt/rocm-5.4.0/
        modules: [rocm/5.4.0]
    rocfft:
      buildable: false
      externals:
      - spec: rocfft@5.4.0
        prefix: /opt/rocm-5.4.0/
        modules: [rocm/5.4.0]
    rocm-clang-ocl:
      buildable: false
      externals:
      - spec: rocm-clang-ocl@5.4.0
        prefix: /opt/rocm-5.4.0/
        modules: [rocm/5.4.0]
    rocm-cmake:
      buildable: false
      externals:
      - spec: rocm-cmake@5.4.0
        prefix: /opt/rocm-5.4.0/
        modules: [rocm/5.4.0]
    rocm-dbgapi:
      buildable: false
      externals:
      - spec: rocm-dbgapi@5.4.0
        prefix: /opt/rocm-5.4.0/
        modules: [rocm/5.4.0]
    rocm-debug-agent:
      buildable: false
      externals:
      - spec: rocm-debug-agent@5.4.0
        prefix: /opt/rocm-5.4.0/
        modules: [rocm/5.4.0]
    rocm-device-libs:
      buildable: false
      externals:
      - spec: rocm-device-libs@5.4.0
        prefix: /opt/rocm-5.4.0/
        modules: [rocm/5.4.0]
    rocm-gdb:
      buildable: false
      externals:
      - spec: rocm-gdb@5.4.0
        prefix: /opt/rocm-5.4.0/
        modules: [rocm/5.4.0]
    rocm-opencl:
      buildable: false
      externals:
      - spec: rocm-opencl@5.4.0
        prefix: /opt/rocm-5.4.0/opencl
        modules: [rocm/5.4.0]
    rocm-smi-lib:
      buildable: false
      externals:
      - spec: rocm-smi-lib@5.4.0
        prefix: /opt/rocm-5.4.0/
        modules: [rocm/5.4.0]
    hip:
      buildable: false
      externals:
      - spec: hip@5.4.0
        prefix: /opt/rocm-5.4.0
        modules:
        - craype-accel-amd-gfx90a
        - rocm/5.4.0
        extra_attributes:
          compilers:
            c: CC
            c++: CC
            hip: CC
        environment:
          set:
            MPICH_GPU_SUPPORT_ENABLED: 1
    llvm-amdgpu:
      buildable: false
      externals:
      - spec: llvm-amdgpu@5.4.0
        prefix: /opt/rocm-5.4.0/llvm
        modules: [rocm/5.4.0]
        extra_attributes:
          compilers:
            c: /opt/rocm-5.4.0/llvm/bin/clang++
            cxx: /opt/rocm-5.4.0/llvm/bin/clang++
    hsakmt-roct:
      buildable: false
      externals:
      - spec: hsakmt-roct@5.4.0
        prefix: /opt/rocm-5.4.0/
        modules: [rocm/5.4.0]
    hsa-rocr-dev:
      buildable: false
      externals:
      - spec: hsa-rocr-dev@5.4.0
        prefix: /opt/rocm-5.4.0/
        modules: [rocm/5.4.0]
        extra_atributes:
          compilers:
            c: /opt/rocm-5.4.0/llvm/bin/clang++
            cxx: /opt/rocm-5.4.0/llvm/bin/clang++
    roctracer-dev-api:
      buildable: false
      externals:
      - spec: roctracer-dev-api@5.4.0
        prefix: /opt/rocm-5.4.0
        modules: [rocm/5.4.0]
    roctracer-dev:
      buildable: false
      externals:
      - spec: roctracer-dev@5.4.0
        prefix: /opt/rocm-5.4.0
        modules: [rocm/5.4.0]
    rocprim:
      buildable: false
      externals:
      - spec: rocprim@5.4.0
        prefix: /opt/rocm-5.4.0
        modules: [rocm/5.4.0]
    rocrand:
      buildable: false
      externals:
      - spec: rocrand@5.4.0
        prefix: /opt/rocm-5.4.0
        modules: [rocm/5.4.0]
    hipsolver:
      buildable: false
      externals:
      - spec: hipsolver@5.4.0
        prefix: /opt/rocm-5.4.0
        modules: [rocm/5.4.0]
    rocsolver:
      buildable: false
      externals:
      - spec: rocsolver@5.4.0
        prefix: /opt/rocm-5.4.0
        modules: [rocm/5.4.0]
    rocsparse:
      buildable: false
      externals:
      - spec: rocsparse@5.4.0
        prefix: /opt/rocm-5.4.0
        modules: [rocm/5.4.0]
    rocthrust:
      buildable: false
      externals:
      - spec: rocthrust@5.4.0
        prefix: /opt/rocm-5.4.0
        modules: [rocm/5.4.0]
    rocprofiler-dev:
      buildable: false
      externals:
      - spec: rocprofiler-dev@5.4.0
        prefix: /opt/rocm-5.4.0
        modules:
        - rocm/5.4.0

  specs:
  # CPU
  - adios
  - aml
  - amrex
  - arborx
  - argobots
  - bolt
  - butterflypack
  - cabana
  - caliper
  - chai ~benchmarks ~tests
  - charliecloud
  - conduit
  - datatransferkit
  - ecp-data-vis-sdk ~cuda ~rocm +adios2 +ascent +cinema +darshan +faodel +hdf5 +paraview +pnetcdf +sz +unifyfs +veloc +visit +vtkm +zfp
  - flecsi
  - flit
  - flux-core
  - fortrilinos
  - gasnet
  - ginkgo
  - globalarrays
  - gmp
  - gotcha
  - h5bench
  - hdf5-vol-async
  - heffte +fftw
  - hpx max_cpu_count=512 networking=mpi
  - hypre
  - kokkos +openmp
  - kokkos-kernels +openmp
  - legion
  - libquo
  - libunwind
  - loki
  - mercury
  - metall
  - mfem
  - mpark-variant
  - mpifileutils ~xattr
  - nccmp
  - nco
  - netlib-scalapack
  - omega-h
  - openpmd-api
  - papi
  - papyrus
  - pdt
  - petsc
  - plumed
  - precice
  - pumi
  - py-libensemble
  - py-petsc4py
  - qthreads scheduler=distrib
  - raja
  - rempi
  - slate ~cuda
  - slepc
  - stc
  - strumpack ~slate
  - sundials
  - superlu
  - superlu-dist
  - swig
  - swig@4.0.2-fortran
  - tasmanian
  - tau +mpi +python ^libelf
  - trilinos +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext
    +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu
    +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos
    +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
  - turbine
  - umap
  - umpire
  - unifyfs
  - upcxx
  - variorum
  - wannier90

  # no amdgpu_target
  - cabana +rocm ^kokkos +rocm amdgpu_target=gfx90a
  - tau +mpi +rocm ^libelf

  # amdgpu_target=gfx90a
  - amrex +rocm amdgpu_target=gfx90a
  - arborx +rocm amdgpu_target=gfx90a
  - caliper +rocm amdgpu_target=gfx90a
  - chai ~benchmarks +rocm amdgpu_target=gfx90a
  - ecp-data-vis-sdk +paraview +vtkm +rocm amdgpu_target=gfx90a # need vtk-m patch: https://github.com/spack/spack/pull/34427/commits (afea4d1c3035830ead2910dd46a517973ee0d477)
  - gasnet +rocm amdgpu_target=gfx90a
  - ginkgo +rocm amdgpu_target=gfx90a
  - heffte +rocm amdgpu_target=gfx90a
  - hpx max_cpu_count=512 +rocm amdgpu_target=gfx90a
  - hypre +rocm amdgpu_target=gfx90a
  - kokkos +rocm amdgpu_target=gfx90a
  - magma ~cuda +rocm amdgpu_target=gfx90a
  - papi +rocm amdgpu_target=gfx90a
  - raja ~openmp +rocm amdgpu_target=gfx90a
  - slate +rocm amdgpu_target=gfx90a
  - strumpack ~slate +rocm amdgpu_target=gfx90a
  - sundials +rocm amdgpu_target=gfx90a
  - superlu-dist +rocm amdgpu_target=gfx90a
  - tasmanian ~openmp +rocm amdgpu_target=gfx90a
  - trilinos@13.4.0 +amesos +amesos2 +anasazi +aztec ~belos +boost +epetra +epetraext
    +ifpack ~ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu
    +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu ~stokhos +stratimikos
    +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
    +rocm amdgpu_target=gfx90a
  - umpire +rocm amdgpu_target=gfx90a
  - upcxx +rocm amdgpu_target=gfx90a

  # INCLUDED IN ECP-DATA-VIS-SDK
  # - adios2
  # - ascent                      # failed because ?
  # - darshan-runtime
  # - darshan-util
  # - faodel
  # - hdf5 +fortran +hl +shared
  # - parallel-netcdf
  # - paraview
  # - sz
  # - veloc
  # - vtk-m
  # - vtk-m ~openmp ~rendering +rocm amdgpu_target=gfx90a  # vtk-m
  # - zfp

  # CPU ERRORS
  # - alquimia                    # pflotran
  # - axom                        # axom
  # - bricks                      # bricks
  # - dealii                      # intel-tbb@2020.3
  # - dyninst                     # intel-tbb@2020.3
  # - exaworks                    # rust
  # - geopm ~doc                  # geopm
  # - hpctoolkit                  # intel-tbb@2020.3
  # - lammps                      # kim-api
  # - libpressio +bitgrooming +bzip2 ~cuda ~cusz +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf  # bitgroomingz
  # - nrm                         # py-scipy@1.5.4
  # - paraview ^llvm              # llvm@14.0.6
  # - parsec ~cuda                # parsec
  # - phist                       # phist
  # - pruners-ninja               # pruners-ninja
  # - py-cinemasci                # py-scipy@1.5.4
  # - py-jupyterhub               # py-jupyterhub
  # - py-warpx ^warpx dims=2      # py-scipy@1.5.4
  # - py-warpx ^warpx dims=3      # py-scipy@1.5.4
  # - py-warpx ^warpx dims=rz     # py-scipy@1.5.4
  # - quantum-espresso            # quantum-espresso
  # - scr                         # scr
  # - stat                        # intel-tbb@2020.3
  # - visit                       # qt@5.14.2
  # --
  # qt@5.14.2: /usr/lib64/gcc/x86_64-suse-linux/7/../../../../x86_64-suse-linux/bin/ld: main.o: in function `main': main.cpp:(.text.startup+0x9): undefined reference to `dlopen'
  # axom: CMake Error at axom/sidre/cmake_install.cmake:154 (file): file INSTALL cannot find "spack-stage-axom-0.7.0-vg7dfgnm4wkwfby5blzlinp25mznp2jr/spack-build-vg7dfgn/lib/fortran/axom_spio.mod": No such file or directory.
  # bitgroomingz: Detecting CXX compiler ABI info - failed
  # bricks: cc1plus: error: bad value ('OFF') for '-mtune=' switch; stencils/3axis_sycl.cpp:8:10: fatal error: 'CL/sycl.hpp' file not found
  # geopm: ftn-2307 ftn: ERROR in command line: The "-m" option must be followed by 0, 1, 2, 3 or 4.
  # intel-tbb@2020.3: clang-14: error: unknown argument: '-flifetime-dse=1'
  # kim-api: ftn-78 ftn: ERROR in command line: The -s option has an invalid argument, "td=gnu".
  # llvm@14.0.6: BitcodeReader.cpp:(.text._ZN4llvm13readWideAPIntENS_8ArrayRefImEEj+0x7f): undefined reference to `__cray_dset_detect'
  # parsec: ftn-78 ftn: ERROR in command line: The -O option has an invalid argument, "g".
  # pflotran: ftn-855 ftn: ERROR GRID_UNSTRUCTURED_MODULE,
  # phist: ftn-78 ftn: ERROR in command line: The -f option has an invalid argument, "finite-math-only".
  # pruners-ninja: https://github.com/spack/spack/issues/32112
  # pruners-ninja: ld.lld: error: duplicate symbol: a
  # py-scipy@1.5.4: setup.py:462: UserWarning: Unrecognized setuptools command ('dist_info --egg-base /tmp/pip-modern-metadata-_bk8h8nt'), proceeding with generating Cython sources and expanding templates
  # qt@5.14.2: /usr/lib64/gcc/x86_64-suse-linux/7/../../../../x86_64-suse-linux/bin/ld: main.cpp:(.text.startup+0x11): undefined reference to `dlclose'
  # qt@5.15.5: /usr/lib64/gcc/x86_64-suse-linux/7/../../../../x86_64-suse-linux/bin/ld: main.cpp:(.text.startup+0x11): undefined reference to `dlclose'
  # quantum-espresso: ftn-100 ftn: ERROR LOCK_BUFFER_RM, File = external/devxlib/src/device_fbuff.f90, Line = 615, Column = 1
  # rust: ld.lld: error: relocation R_X86_64_32 cannot be used against local symbol; recompile with -fPIC
  # rust: ld.lld: error: relocation R_X86_64_32 cannot be used against local symbol; recompile with -fPIC
  # scr: /opt/cray/pe/cce/14.0.0/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld: /opt/cray/pe/mpich/8.1.16/ofi/crayclang/10.0/lib/libmpi_cray.so: undefined reference to `PMI_Barrier'
  # scr: ld: /opt/cray/pe/mpich/8.1.16/ofi/crayclang/10.0/lib/libmpi_cray.so: undefined reference to `PMI_Barrier'

  # GPU ERRORS
  # - hpctoolkit +rocm                                                    # intel-tbb@2020.3
  # - mfem +rocm amdgpu_target=gfx90a  # required adding explicit depends_on for rocsparse  
  # - petsc +rocm amdgpu_target=gfx90a
  # - slepc +rocm amdgpu_target=gfx90a ^petsc +openmp +rocm amdgpu_target=gfx90a # need +openmp due to cce@14.0.2 bug involving craype-accel-amd-gfx90a
  # - vtk-m +rocm amdgpu_target=gfx90a                                    # vtk-m@1.9.0
  # --
  # intel-tbb@2020.3: clang-14: error: unknown argument: '-flifetime-dse=1'

  # SKIPPED
  # - archer ^libelf        # part of llvm now
  # - gptune ~mpispawn      # hard dependency on openmpi
  # - libnrm                # hard dependency on mpich
  # - nvhpc                 # not relevant
  # - openmpi               # not relevant
  # - plasma                # %cce conflict