# Invocation command line: # /lfs/lfs09/emelnich/SPECMPI2007/mpi2007-skl-qual-opa-avx512/bin/runspec --reportable --config intel_mpi2007.avx512.fast.cfg --define wd=/lfs/lfs09/emelnich/SPECMPI2007/mpi2007-skl-qual-opa-avx512 --define expid=SUBMISSION/SKX-GOLD-6148.2.OPA-10.7.avx512.comp_2018u0.mpi_2019.0.004_ww28_20180713/medium_mref.TURBO_ON.default/FI_PSM2_DELAY_0.Libfabric_internal.FABRIC_shm:ofi/ --define fabric=shm:ofi --define ppn=40 --flagsurl EM64T_Intel160_flags.xml --size mref --iterations 3 -I --output_format=txt --ranks=640 medium # output_root was not used for this run ############################################################################ ##################################################################### # # Config file to run SPEC MPI2007 with Intel Software Toolchain # (Intel Compiler and Intel MPI) # ##################################################################### env_vars = 1 basepeak = 1 reportable = 0 ignore_errors = 1 iterations = 3 makeflags = -j 16 tune = base #size = mref use_version_url = 1 version_url = version.txt use_submit_for_speed = 1 output_format = all expid = %{expid} %ifdef %{fabric} FABRIC=%{fabric} %else FABRIC=shm:tmi %endif FC = mpiifort CC = mpiicc CXX = mpiicpc ##################################################################### # Portability flags ##################################################################### 121.pop2=default=default=default: CPORTABILITY = -DSPEC_MPI_CASE_FLAG 126.lammps=default=default=default: CXXPORTABILITY = -DMPICH_IGNORE_CXX_SEEK 127.wrf2=default=default=default: CPORTABILITY = -DSPEC_MPI_CASE_FLAG -DSPEC_MPI_LINUX 129.tera_tf=default=default=default: srcalt=add_rank_support 130.socorro=default=default=default: srcalt=nullify_ptrs FPORTABILITY = -assume nostd_intent_in 143.dleslie=default=default=default: srcalt=integer_overflow ################################################################# # Optimization flags ################################################################# default=default=default=default: OPTIMIZE = -O3 -xCORE-AVX512 -no-prec-div -ipo submit = mpiexec.hydra -f \$LSB_DJOB_HOSTFILE -genv I_MPI_FABRICS %{fabric} -genv I_MPI_PROVIDER psm2 -genv I_MPI_FALLBACK 0 -genv I_MPI_COMPATIBILITY=3 -n $ranks -ppn %{ppn} $command ################################################################## ## Notes ################################################################## test_sponsor = Intel Corporation license_num = 13 tester = Intel Corporation test_date = Jul-2017 hw_avail = Aug-2018 sw_avail = Jul-2017 prepared_by = Egor Melnichenko system_vendor = Intel Corporation system_name000 = Endeavor (Intel Xeon Gold 6148, 2.40 GHz, system_name001 = DDR4-2666 MHz, SMT on, Turbo on) ## ## Computation node info ## node_compute_count = 16 node_compute_hw_adapter_ib_count = 1 node_compute_hw_adapter_ib_data_rate = 12.5 GB/s node_compute_hw_adapter_ib_driver = IFS 10.7 node_compute_hw_adapter_ib_firmware = 1.26.1 node_compute_hw_adapter_ib_interconnect = Intel Omni-Path Fabric 100 series node_compute_hw_adapter_ib_model = Intel Omni-Path Edge Switch 100 series node_compute_hw_adapter_ib_ports_used = 1 node_compute_hw_adapter_ib_slot_type = PCI-Express x16 node_compute_hw_cpu_char000 = Intel Turbo Boost Technology up to 3.7 GHz node_compute_hw_cpu_mhz = 2400 node_compute_hw_cpu_name = Intel Xeon Gold 6148 node_compute_hw_disk = ATA INTEL SSDSC2BA80 node_compute_hw_memory = 192 GB (16 x 12 GB 2Rx4 DDR4-2666) node_compute_hw_model = Intel Server System R2208WFTZS node_compute_hw_nchips = 2 node_compute_hw_ncores = 40 node_compute_hw_ncoresperchip = 20 node_compute_hw_ncpuorder = 1-2 chips node_compute_hw_nthreadspercore = 2 node_compute_hw_ocache = None node_compute_hw_other = None node_compute_hw_pcache = 32 KB I + 32 KB D on chip per core node_compute_hw_scache = 1 MB I+D on chip per core node_compute_hw_tcache000 = 27.5 MB I+D on chip per chip node_compute_hw_vendor = Intel node_compute_label = Intel Server System R2208WFTZS node_compute_order = 1 node_compute_purpose = Compute node_compute_sw_localfile = Linux/xfs node_compute_sw_os = Oracle Linux Server release 7.4 node_compute_sw_other = IBM Platform LSF Standard 9.1.1.1 node_compute_sw_sharedfile = Lustre FS node_compute_sw_state = Multi-User ## ## Fileserver node info ## node_fileserver_count = 11 node_fileserver_hw_adapter_fs_count = 1 node_fileserver_hw_adapter_fs_data_rate = 12.5 GB/s node_fileserver_hw_adapter_fs_driver = IFS 10.7 node_fileserver_hw_adapter_fs_firmware = 1.26.1 node_fileserver_hw_adapter_fs_interconnect = Intel Omni-Path Fabric 100 series node_fileserver_hw_adapter_fs_model000 = Intel Omni-Path Fabric Adapter 100 series node_fileserver_hw_adapter_fs_ports_used = 1 node_fileserver_hw_adapter_fs_slot_type = PCI-Express x16 node_fileserver_hw_cpu_mhz = 2700 node_fileserver_hw_cpu_name = Intel Xeon E5-2680 node_fileserver_hw_disk = 136 TB 3 RAID with 8 SAS/SATA node_fileserver_hw_memory = 64 GB per node (8 x 8 GB 1600MHz Reg ECC DDR3) node_fileserver_hw_model = Intel Server System R2208GZ4GC4 node_fileserver_hw_nchips = 2 node_fileserver_hw_ncores = 16 node_fileserver_hw_ncoresperchip = 8 node_fileserver_hw_ncpuorder = 1-2 chips node_fileserver_hw_nthreadspercore = 2 node_fileserver_hw_ocache = None node_fileserver_hw_other = None node_fileserver_hw_pcache = 32 KB I + 32 KB D on chip per core node_fileserver_hw_scache = 2 MB I+D on chip per chip node_fileserver_hw_tcache = None node_fileserver_hw_vendor = Intel node_fileserver_label = Lustre FS node_fileserver_order = 2 node_fileserver_purpose = Fileserver node_fileserver_sw_localfile = None node_fileserver_sw_os = Redhat Enterprise Linux Server Release 7.4 node_fileserver_sw_other = None node_fileserver_sw_sharedfile = Lustre FS node_fileserver_sw_state = Multi-User ## ## IB interconnect ## interconnect_ib_hw_model = Intel Omni-Path Fabric 100 series interconnect_ib_hw_switch_3600_count = 24 interconnect_ib_hw_switch_3600_data_rate = 12.5 GB/s interconnect_ib_hw_switch_3600_firmware = 1.26.1 interconnect_ib_hw_switch_3600_model = Intel Omni-Path Edge Switch 100 series interconnect_ib_hw_switch_3600_ports = 48 interconnect_ib_hw_topo = Fat tree interconnect_ib_hw_vendor = Intel interconnect_ib_label = Intel Omni-Path 100 series interconnect_ib_order = 1 interconnect_ib_purpose = MPI and I/O traffic ## ## Cluster file system interconnect ## ## ## Hardware ## system_class = Homogeneous max_peak_ranks = -- max_ranks = 80 ## ## Software ## sw_avail = Nov-2018 sw_base_ptrsize = 64-bit sw_c_compiler000 = Intel C++ Composer XE 2018 for Linux sw_c_compiler001 = Version 18.0.0 Build 20170811 sw_cxx_compiler000 = Intel C++ Composer XE 2018 for Linux sw_cxx_compiler001 = Version 18.0.0 Build 20170811 sw_f_compiler000 = Intel Fortran Composer XE 2018 for Linux sw_f_compiler001 = Version 18.0.0 Build 20170811 sw_mpi_library = Intel MPI Library 2019 Build 20180829 sw_mpi_other = libfabric-1.6.1 sw_other = None sw_parallel_other = -- sw_peak_ptrsize = 64-bit sw_preprocessors = No sw_base_ptrsize = 64-bit ## ## General notes ## #notes_000 = 143.dleslie (base) = "integer_overflow" src.alt was used. #notes_005 = #notes_010 = 129.tera_tf (base) = "add_rank_support" src.alt was used. #notes_015 = notes_020 = MPI startup command: notes_025 = mpiexec.hydra command was used to start MPI jobs. notes_030 = export I_MPI_FABRICS=shm:ofi notes_035 = export FI_PSM2_INJECT_SIZE=8192 notes_040 = export I_MPI_PIN_DOMAIN=core notes_045 = export I_MPI_PIN_ORDER=bunch notes_050 = export FI_PSM2_DELAY=0 notes_055 = export FI_PSM2_LAZY_CONN=1 notes_060 = export I_MPI_COMPATIBILITY=3 notes_065 = Spectre & Meltdown: notes_070 = Kernel: 3.10.0-862.11.6.el7.crt1.x86_64 notes_075 = Microcode: 0x200004d notes_080 = l1tf: Mitigation: PTE Inversion notes_085 = meltdown: Mitigation: PTI notes_090 = spec_store_bypass: Mitigation: Speculative Store Bypass disabled via prctl and seccomp notes_095 = spectre_v1: Mitigation: Load fences, __user pointer sanitization notes_100 = spectre_v2: Mitigation: IBRS (kernel) notes_105 = BIOS settings: notes_110 = Intel Hyper-Threading Technology (SMT) = Enabled (default is Enabled) notes_115 = Intel Turbo Boost Technology (Turbo) = Enabled (default is Enabled) notes_120 = RAM configuration: notes_125 = Compute nodes have 2x16-GB RDIMM on each memory channel. notes_130 = Network: notes_135 = Endeavour Omni-Path Fabric consists of 48-port switches = 24 core switches notes_140 = connected to each leaf of the rack switch. notes_145 = HFI driver parameters: notes_150 = cache_size = 1024 notes_155 = rcvhdrcnt = 4096 notes_160 = Job placement: notes_165 = Each MPI job was assigned to a topologically compact set of nodes, i.e. notes_170 = the minimal needed number of leaf switches was used for each job = 1 switch notes_175 = for 40/80/160/320/640 ranks, 2 switches for 1280 and 1980 ranks. notes_180 = IBM Platform LSF was used for job submission. It has no impact on performance. notes_185 = Information can be found at: http://www.ibm.com # The following section was added automatically, and contains settings that # did not appear in the original configuration file, but were added to the # raw file after the run. default: flagsurl000 = http://www.spec.org/mpi2007/flags/EM64T_Intel140_flags.20190110.xml notes_000 = 130.socorro (base): "nullify_ptrs" src.alt was used. notes_005 = 129.tera_tf (base): "add_rank_support" src.alt was used. notes_010 = 143.dleslie (base): "integer_overflow" src.alt was used. notes_015 = node_fileserver_hw_cpu_char000 = Intel Turbo Boost Technology up to 3.5 GHz