From c21126849163a53575f8c1a0715054c6d02fe929 Mon Sep 17 00:00:00 2001 From: Andrea Valassi Date: Tue, 30 Nov 2021 17:11:06 +0100 Subject: [PATCH 01/34] [klas5X] remove old code from pIparIp4Ipag --- epochX/cudacpp/ee_mumu/src/HelAmps_sm.cc | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/epochX/cudacpp/ee_mumu/src/HelAmps_sm.cc b/epochX/cudacpp/ee_mumu/src/HelAmps_sm.cc index a1cdb5ab30..3b0d9a16c6 100644 --- a/epochX/cudacpp/ee_mumu/src/HelAmps_sm.cc +++ b/epochX/cudacpp/ee_mumu/src/HelAmps_sm.cc @@ -49,16 +49,6 @@ namespace MG5_sm const int ip4, const int ipagM ) { - /* - //#ifndef MGONGPU_CPPSIMD - // TEMPORARY during epochX step3! Eventually remove this section (start) - // TEMPORARY during epochX step3! THERE IS NO SIMD YET: HENCE ipagM=ievt - // NB: this is needed for neppM>1 while neppV==1 (no SIMD) in eemumu.auto - // NB: this remains valid for neppM==1 with neppV==1 in eemumu (scalar) - return pIparIp4Ievt( momenta, ipar, ip4, ipagM ); - // TEMPORARY during epochX step3! Eventually remove this section (end) - //#else - */ // NB: this assumes that neppV == neppM! // NB: this is the same as "pIparIp4Ievt( momenta, ipar, ip4, ipagM )" for neppM==1 with neppV==1 using mgOnGpu::np4; From 0b2a6ac4e854b77b2fd7487506b86af6e5fbb7c6 Mon Sep 17 00:00:00 2001 From: Andrea Valassi Date: Tue, 30 Nov 2021 17:31:41 +0100 Subject: [PATCH 02/34] [klas5X] bug fix in operator<< for vector types (replace std::cout by argument out) --- epochX/cudacpp/ee_mumu/src/mgOnGpuVectors.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/epochX/cudacpp/ee_mumu/src/mgOnGpuVectors.h b/epochX/cudacpp/ee_mumu/src/mgOnGpuVectors.h index e1da365663..9e8b34988f 100644 --- a/epochX/cudacpp/ee_mumu/src/mgOnGpuVectors.h +++ b/epochX/cudacpp/ee_mumu/src/mgOnGpuVectors.h @@ -114,7 +114,7 @@ using mgOnGpu::bool_v; inline std::ostream& operator<<( std::ostream& out, const bool_v& v ) { out << "{ " << v[0]; - for ( int i=1; i Date: Tue, 30 Nov 2021 17:36:35 +0100 Subject: [PATCH 03/34] [klas5X] do not filter out WARNING in throughputX.sh --- epochX/cudacpp/tput/throughputX.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/epochX/cudacpp/tput/throughputX.sh b/epochX/cudacpp/tput/throughputX.sh index d3ff1fbd85..985614c489 100755 --- a/epochX/cudacpp/tput/throughputX.sh +++ b/epochX/cudacpp/tput/throughputX.sh @@ -240,6 +240,7 @@ function runExe() { pattern="${pattern}|CUCOMPLEX" pattern="${pattern}|COMMON RANDOM" pattern="${pattern}|ERROR" + pattern="${pattern}|WARNING" # TEMPORARY! OLD C++/CUDA CODE (START) pattern="${pattern}|EvtsPerSec\[Matrix" # TEMPORARY! OLD C++/CUDA CODE (END) From 98008b40c3a0b3456877e697230a62abfca77ca8 Mon Sep 17 00:00:00 2001 From: Andrea Valassi Date: Tue, 30 Nov 2021 18:32:17 +0100 Subject: [PATCH 04/34] [klas5X] COMPLETE the port of the latest klas5 in epoch1/eemumu to klas5X in epochX/eemumu Ported everything as-is (in one single commit), including stuff that eventually needs to be cleaned up (e.g. it will probably be better to only support some given choices of neppM and neppV and fail otherwise) This essentially completes (except for backports) issue #176 about removing neppV from the public API... Performance (see next commit) is almost the same. Probably 512y eemumu loses 1% because I have removed a few vectorizations on allMEs. Maybe I can get back a tiny bit even there. --- epochX/cudacpp/ee_mumu/SubProcesses/Memory.h | 19 +- .../P1_Sigma_sm_epem_mupmum/CPPProcess.cc | 82 ++++++--- .../P1_Sigma_sm_epem_mupmum/CPPProcess.h | 14 +- .../P1_Sigma_sm_epem_mupmum/check_sa.cc | 35 +--- .../cudacpp/ee_mumu/SubProcesses/runTest.cc | 36 ++-- .../cudacpp/ee_mumu/SubProcesses/testxxx.cc | 8 +- epochX/cudacpp/ee_mumu/src/HelAmps_sm.cc | 173 +++++++++++++++--- epochX/cudacpp/ee_mumu/src/HelAmps_sm.h | 20 +- epochX/cudacpp/ee_mumu/src/mgOnGpuConfig.h | 84 ++++++--- epochX/cudacpp/ee_mumu/src/mgOnGpuVectors.h | 8 +- epochX/cudacpp/ee_mumu/src/rambo.cc | 6 +- epochX/cudacpp/ee_mumu/src/rambo.h | 6 +- 12 files changed, 331 insertions(+), 160 deletions(-) diff --git a/epochX/cudacpp/ee_mumu/SubProcesses/Memory.h b/epochX/cudacpp/ee_mumu/SubProcesses/Memory.h index d7d1c7d4da..a9c08d3dd0 100644 --- a/epochX/cudacpp/ee_mumu/SubProcesses/Memory.h +++ b/epochX/cudacpp/ee_mumu/SubProcesses/Memory.h @@ -1,8 +1,9 @@ /* - * memory.h + * Memory.h * * Created on: 19.11.2020 * Author: shageboeck + * Modified: avalassi (vectors, alignment) */ #ifndef MEMORY_H @@ -46,13 +47,25 @@ std::unique_ptr> hstMakeUnique(std::size_t N) { #else +template +struct CppHstDeleter { + void operator()(T* mem) { + ::operator delete( mem, std::align_val_t{ mgOnGpu::cppAlign } ); + } +}; + template inline -std::unique_ptr hstMakeUnique(std::size_t N) { return std::unique_ptr{ new T[N]() }; }; +std::unique_ptr> hstMakeUnique(std::size_t N) { + // See https://www.bfilipek.com/2019/08/newnew-align.html#requesting-different-alignment + return std::unique_ptr>{ new( std::align_val_t{ mgOnGpu::cppAlign } ) T[N]() }; +}; #ifdef MGONGPU_CPPSIMD template<> inline -std::unique_ptr hstMakeUnique(std::size_t N) { return std::unique_ptr{ new fptype_v[N/neppV]() }; }; +std::unique_ptr> hstMakeUnique(std::size_t N) { + return std::unique_ptr>{ new( std::align_val_t{ mgOnGpu::cppAlign } ) fptype_v[N/neppV]() }; +}; #endif diff --git a/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc b/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc index 906c8523e2..d49bf181ce 100644 --- a/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc +++ b/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc @@ -71,10 +71,10 @@ namespace Proc __device__ INLINE void calculate_wavefunctions( int ihel, - const fptype_sv* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM - fptype_sv* allMEs // output: allMEs[npagM][neppM], final |M|^2 averaged over helicities + const fptype* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM + fptype* allMEs // output: allMEs[nevt], |M|^2 running_sum_over_helicities #ifndef __CUDACC__ - , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) #endif ) //ALWAYS_INLINE // attributes are not permitted in a function definition @@ -99,6 +99,9 @@ namespace Proc // === Calculate wavefunctions and amplitudes for all diagrams in all processes - Loop over nevt events === #ifndef __CUDACC__ const int npagV = nevt / neppV; +#ifdef MGONGPU_CPPSIMD + const bool isAligned_allMEs = ( (size_t)(allMEs) % mgOnGpu::cppAlign == 0 ); // require SIMD-friendly alignment by at least neppV*sizeof(fptype) +#endif // ** START LOOP ON IPAGV ** #ifdef _OPENMP // (NB gcc9 or higher, or clang, is required) @@ -106,7 +109,11 @@ namespace Proc // - shared: as the name says // - private: give each thread its own copy, without initialising // - firstprivate: give each thread its own copy, and initialise with value from outside +#ifdef MGONGPU_CPPSIMD +#pragma omp parallel for default(none) shared(allmomenta,allMEs,cHel,cIPC,cIPD,ihel,npagV,isAligned_allMEs) private (amp_sv,w_sv,jamp_sv) +#else #pragma omp parallel for default(none) shared(allmomenta,allMEs,cHel,cIPC,cIPD,ihel,npagV) private (amp_sv,w_sv,jamp_sv) +#endif #endif for ( int ipagV = 0; ipagV < npagV; ++ipagV ) #endif @@ -193,13 +200,27 @@ namespace Proc #ifdef __CUDACC__ const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid allMEs[ievt] += deltaMEs; - //printf( "calculate_wavefunction: %6d %2d %f\n", ievt, ihel, allMEs[ievt] ); + //if ( cNGoodHel > 0 ) printf( "calculate_wavefunction: %6d %2d %f\n", ievt, ihel, allMEs[ievt] ); +#else +#ifdef MGONGPU_CPPSIMD + if ( isAligned_allMEs ) + { + *reinterpret_cast( &( allMEs[ipagV*neppV] ) ) += deltaMEs; + } + else + { + for ( int ieppV=0; ieppV 0 ) + // for ( int ieppV=0; ieppV 0 ) printf( "calculate_wavefunction: %6d %2d %f\n", ipagV, ihel, allMEs[ipagV] ); +#endif #endif } - mgDebug( 1, __FUNCTION__ ); return; } @@ -243,10 +264,13 @@ namespace Proc memcpy( cHel, tHel, ncomb * mgOnGpu::npar * sizeof(short) ); #endif // SANITY CHECK: GPU memory usage may be based on casts of fptype[2] to cxtype - assert( sizeof(cxtype) == 2 * sizeof(fptype) ); + static_assert( sizeof(cxtype) == 2 * sizeof(fptype) ); #ifndef __CUDACC__ - // SANITY CHECK: momenta AOSOA uses vectors with the same size as fptype_v - assert( neppV == mgOnGpu::neppM ); + // SANITY CHECK: check that neppR, neppM and neppV are powers of two (https://stackoverflow.com/a/108360) + auto ispoweroftwo = []( int n ) { return ( n > 0 ) && !( n & ( n - 1 ) ); }; + static_assert( ispoweroftwo( mgOnGpu::neppR ) ); + static_assert( ispoweroftwo( mgOnGpu::neppM ) ); + static_assert( ispoweroftwo( neppV ) ); #endif } @@ -356,9 +380,9 @@ namespace Proc #ifdef __CUDACC__ __global__ - void sigmaKin_getGoodHel( const fptype_sv* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM - fptype_sv* allMEs, // output: allMEs[npagM][neppM], final |M|^2 averaged over helicities - bool* isGoodHel ) // output: isGoodHel[ncomb] - device array + void sigmaKin_getGoodHel( const fptype* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM + fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities + bool* isGoodHel ) // output: isGoodHel[ncomb] - device array { const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid // FIXME: assume process.nprocesses == 1 for the moment (eventually: need a loop over processes here?) @@ -376,33 +400,35 @@ namespace Proc } } #else - void sigmaKin_getGoodHel( const fptype_sv* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM - fptype_sv* allMEs, // output: allMEs[npagM][neppM], final |M|^2 averaged over helicities - bool* isGoodHel // output: isGoodHel[ncomb] - device array - , const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + void sigmaKin_getGoodHel( const fptype* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM + fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities + bool* isGoodHel // output: isGoodHel[ncomb] - device array + , const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) { + assert( (size_t)(allmomenta) % mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment + //assert( (size_t)(allMEs) % mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment const int maxtry0 = ( neppV > 16 ? neppV : 16 ); // 16, but at least neppV (otherwise the npagV loop does not even start) - fptype_sv allMEsLast[maxtry0/neppV] = { 0 }; + fptype allMEsLast[maxtry0] = { 0 }; const int maxtry = std::min( maxtry0, nevt ); // 16, but at most nevt (avoid invalid memory access if nevt allMEs[nevt*nprocesses]?) __global__ - void sigmaKin( const fptype_sv* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM - fptype_sv* allMEs // output: allMEs[npagM][neppM], final |M|^2 averaged over helicities + void sigmaKin( const fptype* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM + fptype* allMEs // output: allMEs[nevt], |M|^2 final_avg_over_helicities #ifndef __CUDACC__ - , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) #endif ) { @@ -472,7 +498,8 @@ namespace Proc const int npagV = nevt/neppV; for ( int ipagV = 0; ipagV < npagV; ++ipagV ) { - allMEs[ipagV] = fptype_sv{ 0 }; + for ( int ieppV=0; ieppV( nRnarray ); // AOSOA[npagR][nparf][np4][neppR] (NB: nevt=npagR*neppR) + auto hstRnarray = hstMakeUnique( nRnarray ); // AOSOA[npagR][nparf][np4][neppR] (NB: nevt=npagR*neppR) #endif - auto hstMomenta = hstMakeUnique( nMomenta ); // AOSOA[npagM][npar][np4][neppM] (NB: nevt=npagM*neppM) - auto hstIsGoodHel = hstMakeUnique( ncomb ); - auto hstWeights = hstMakeUnique( nWeights ); - auto hstMEs = hstMakeUnique( nMEs ); // AOSOA[npagM][neppM] (NB: nevt=npagM*neppM) + auto hstMomenta = hstMakeUnique( nMomenta ); // AOSOA[npagM][npar][np4][neppM] (NB: nevt=npagM*neppM) + auto hstIsGoodHel = hstMakeUnique( ncomb ); + auto hstWeights = hstMakeUnique( nWeights ); + auto hstMEs = hstMakeUnique( nMEs ); // ARRAY[nevt] #ifdef __CUDACC__ - auto devRnarray = devMakeUnique( nRnarray ); // AOSOA[npagR][nparf][np4][neppR] (NB: nevt=npagR*neppR) - auto devMomenta = devMakeUnique( nMomenta ); // AOSOA[npagM][npar][np4][neppM] (NB: nevt=npagM*neppM) - auto devIsGoodHel = devMakeUnique( ncomb ); - auto devWeights = devMakeUnique( nWeights ); - auto devMEs = devMakeUnique( nMEs ); // AOSOA[npagM][neppM] (NB: nevt=npagM*neppM) + auto devRnarray = devMakeUnique( nRnarray ); // AOSOA[npagR][nparf][np4][neppR] (NB: nevt=npagR*neppR) + auto devMomenta = devMakeUnique( nMomenta ); // AOSOA[npagM][npar][np4][neppM] (NB: nevt=npagM*neppM) + auto devIsGoodHel = devMakeUnique( ncomb ); + auto devWeights = devMakeUnique( nWeights ); + auto devMEs = devMakeUnique( nMEs ); // ARRAY[nevt] #if defined MGONGPU_CURAND_ONHOST or defined MGONGPU_COMMONRAND_ONHOST const int nbytesRnarray = nRnarray * sizeof(fptype); @@ -553,37 +553,22 @@ int main(int argc, char **argv) // NB: 'setw' affects only the next field (of any type) std::cout << std::scientific // fixed format: affects all floats (default precision: 6) << std::setw(4) << ipar + 1 -#ifndef MGONGPU_CPPSIMD << std::setw(14) << hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + 0*neppM + ieppM] // AOSOA[ipagM][ipar][0][ieppM] << std::setw(14) << hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + 1*neppM + ieppM] // AOSOA[ipagM][ipar][1][ieppM] << std::setw(14) << hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + 2*neppM + ieppM] // AOSOA[ipagM][ipar][2][ieppM] << std::setw(14) << hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + 3*neppM + ieppM] // AOSOA[ipagM][ipar][3][ieppM] -#else - << std::setw(14) << hstMomenta[ipagM*npar*np4 + ipar*np4 + 0][ieppM] // AOSOA[ipagM][ipar][0][ieppM] - << std::setw(14) << hstMomenta[ipagM*npar*np4 + ipar*np4 + 1][ieppM] // AOSOA[ipagM][ipar][1][ieppM] - << std::setw(14) << hstMomenta[ipagM*npar*np4 + ipar*np4 + 2][ieppM] // AOSOA[ipagM][ipar][2][ieppM] - << std::setw(14) << hstMomenta[ipagM*npar*np4 + ipar*np4 + 3][ieppM] // AOSOA[ipagM][ipar][3][ieppM] -#endif << std::endl << std::defaultfloat; // default format: affects all floats } std::cout << std::string(SEP79, '-') << std::endl; // Display matrix elements std::cout << " Matrix element = " -#ifndef MGONGPU_CPPSIMD << hstMEs[ievt] -#else - << hstMEs[ievt/neppM][ievt%neppM] -#endif << " GeV^" << meGeVexponent << std::endl; // FIXME: assume process.nprocesses == 1 std::cout << std::string(SEP79, '-') << std::endl; } // Fill the arrays with ALL MEs and weights -#ifndef MGONGPU_CPPSIMD matrixelementALL[iiter*nevt + ievt] = hstMEs[ievt]; // FIXME: assume process.nprocesses == 1 -#else - matrixelementALL[iiter*nevt + ievt] = hstMEs[ievt/neppM][ievt%neppM]; // FIXME: assume process.nprocesses == 1 -#endif weightALL[iiter*nevt + ievt] = hstWeights[ievt]; } diff --git a/epochX/cudacpp/ee_mumu/SubProcesses/runTest.cc b/epochX/cudacpp/ee_mumu/SubProcesses/runTest.cc index d92ffe4e7a..a4f3519434 100644 --- a/epochX/cudacpp/ee_mumu/SubProcesses/runTest.cc +++ b/epochX/cudacpp/ee_mumu/SubProcesses/runTest.cc @@ -21,7 +21,7 @@ template using unique_ptr_host = std::unique_ptr>; #else template -using unique_ptr_host = std::unique_ptr; +using unique_ptr_host = std::unique_ptr>; #endif struct CUDA_CPU_TestBase : public TestDriverBase { @@ -47,11 +47,11 @@ struct CPUTest : public CUDA_CPU_TestBase { // Struct data members (process, and memory structures for random numbers, momenta, matrix elements and weights on host and device) // [NB the hst/dev memory arrays must be initialised in the constructor, see issue #290] Proc::CPPProcess process; - unique_ptr_host hstRnarray; - unique_ptr_host hstMomenta; - unique_ptr_host hstIsGoodHel; - unique_ptr_host hstWeights; - unique_ptr_host hstMEs; + unique_ptr_host hstRnarray; + unique_ptr_host hstMomenta; + unique_ptr_host hstIsGoodHel; + unique_ptr_host hstWeights; + unique_ptr_host hstMEs; // Create a process object // Read param_card and set parameters @@ -61,11 +61,11 @@ struct CPUTest : public CUDA_CPU_TestBase { CPUTest( const std::string& refFileName ) : CUDA_CPU_TestBase( refFileName ), process(niter, gpublocks, gputhreads, /*verbose=*/false), - hstRnarray { hstMakeUnique( nRnarray ) }, // AOSOA[npagR][nparf][np4][neppR] - hstMomenta { hstMakeUnique( nMomenta ) }, // AOSOA[npagM][npar][np4][neppM] - hstIsGoodHel{ hstMakeUnique( mgOnGpu::ncomb ) }, - hstWeights { hstMakeUnique( nWeights ) }, - hstMEs { hstMakeUnique( nMEs ) } // AOSOA[npagM][neppM] + hstRnarray { hstMakeUnique( nRnarray ) }, // AOSOA[npagR][nparf][np4][neppR] + hstMomenta { hstMakeUnique( nMomenta ) }, // AOSOA[npagM][npar][np4][neppM] + hstIsGoodHel{ hstMakeUnique( mgOnGpu::ncomb ) }, + hstWeights { hstMakeUnique( nWeights ) }, + hstMEs { hstMakeUnique( nMEs ) } // ARRAY[nevt] { process.initProc("../../Cards/param_card.dat"); } @@ -106,19 +106,11 @@ struct CPUTest : public CUDA_CPU_TestBase { assert(particle < npar); const auto ipagM = evtNo / neppM; // #eventpage in this iteration const auto ieppM = evtNo % neppM; // #event in the current eventpage in this iteration -#ifndef MGONGPU_CPPSIMD return hstMomenta[ipagM*npar*np4*neppM + particle*np4*neppM + component*neppM + ieppM]; -#else - return hstMomenta[ipagM*npar*np4 + particle*np4 + component][ieppM]; -#endif }; fptype getMatrixElement(std::size_t ievt) const override { -#ifndef MGONGPU_CPPSIMD return hstMEs[ievt]; -#else - return hstMEs[ievt/neppV][ievt%neppV]; -#endif } }; @@ -161,12 +153,12 @@ struct CUDATest : public CUDA_CPU_TestBase { hstMomenta { hstMakeUnique( nMomenta ) }, // AOSOA[npagM][npar][np4][neppM] (nevt=npagM*neppM) hstIsGoodHel{ hstMakeUnique( mgOnGpu::ncomb ) }, hstWeights { hstMakeUnique( nWeights ) }, - hstMEs { hstMakeUnique( nMEs ) }, + hstMEs { hstMakeUnique( nMEs ) }, // ARRAY[nevt] devRnarray { devMakeUnique( nRnarray ) }, // AOSOA[npagR][nparf][np4][neppR] (nevt=npagR*neppR) - devMomenta { devMakeUnique( nMomenta ) }, + devMomenta { devMakeUnique( nMomenta ) }, // AOSOA[npagM][npar][np4][neppM] (nevt=npagM*neppM) devIsGoodHel{ devMakeUnique( mgOnGpu::ncomb ) }, devWeights { devMakeUnique( nWeights ) }, - devMEs { devMakeUnique( nMEs ) } + devMEs { devMakeUnique( nMEs ) } // ARRAY[nevt] { process.initProc("../../Cards/param_card.dat"); } diff --git a/epochX/cudacpp/ee_mumu/SubProcesses/testxxx.cc b/epochX/cudacpp/ee_mumu/SubProcesses/testxxx.cc index 6df896c75f..9ae06a7404 100644 --- a/epochX/cudacpp/ee_mumu/SubProcesses/testxxx.cc +++ b/epochX/cudacpp/ee_mumu/SubProcesses/testxxx.cc @@ -31,8 +31,8 @@ TEST( XTESTID_CPU( MG_EPOCH_PROCESS_ID ), testxxx ) assert( nevt % neppM == 0 ); // nevt must be a multiple of neppM // Fill in the input momenta const int nMomenta = np4 * npar * nevt; - auto hstMomenta = hstMakeUnique( nMomenta ); // AOSOA[npagM][npar=4][np4=4][neppM] - const fptype par0[np4 * nevt] = // AOS[nevt][np4] + auto hstMomenta = hstMakeUnique( nMomenta ); // AOSOA[npagM][npar=4][np4=4][neppM] + const fptype par0[np4 * nevt] = // AOS[nevt][np4] { 500, 0, 0, 500, // #0 (m=0 pT=0 E=pz>0) 500, 0, 0, -500, // #1 (m=0 pT=0 -E=pz<0) 500, 300, 400, 0, // #2 (m=0 pT>0 pz=0) @@ -72,11 +72,7 @@ TEST( XTESTID_CPU( MG_EPOCH_PROCESS_ID ), testxxx ) { const int ipagM = ievt/neppM; // #eventpage in this iteration const int ieppM = ievt%neppM; // #event in the current eventpage in this iteration -#ifdef MGONGPU_CPPSIMD - hstMomenta[ipagM*npar*np4 + ipar*np4 + ip4][ieppM] = par0[ievt*np4 + ip4]; // AOS to AOSOA -#else hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + ip4*neppM + ieppM] = par0[ievt*np4 + ip4]; // AOS to AOSOA -#endif } } // Expected output wavefunctions diff --git a/epochX/cudacpp/ee_mumu/src/HelAmps_sm.cc b/epochX/cudacpp/ee_mumu/src/HelAmps_sm.cc index 3b0d9a16c6..3ec3d545c3 100644 --- a/epochX/cudacpp/ee_mumu/src/HelAmps_sm.cc +++ b/epochX/cudacpp/ee_mumu/src/HelAmps_sm.cc @@ -23,8 +23,8 @@ namespace MG5_sm //-------------------------------------------------------------------------- -#ifdef __CUDACC__ - // Return by reference + // Decode momentum AOSOA: compute address of fptype for the given particle, 4-momentum component and event + // Return the fptype by reference (equivalent to returning its memory address) __device__ inline const fptype& pIparIp4Ievt( const fptype* momenta, // input: momenta as AOSOA[npagM][npar][4][neppM] const int ipar, @@ -33,28 +33,149 @@ namespace MG5_sm { using mgOnGpu::np4; using mgOnGpu::npar; - const int neppM = mgOnGpu::neppM; // AOSOA layout: constant at compile-time - const int ipagM = ievt/neppM; // #eventpage in this iteration - const int ieppM = ievt%neppM; // #event in the current eventpage in this iteration - //printf( "%f\n", momenta[ipagM*npar*np4*neppM + ipar*np4*neppM + ip4*neppM + ieppM] ); + constexpr int neppM = mgOnGpu::neppM; // AOSOA layout: constant at compile-time + const int ipagM = ievt/neppM; // #event "M-page" + const int ieppM = ievt%neppM; // #event in the current event M-page + //printf( "%2d %2d %8d %8.3f\n", ipar, ip4, ievt, momenta[ipagM*npar*np4*neppM + ipar*np4*neppM + ip4*neppM + ieppM] ); return momenta[ipagM*npar*np4*neppM + ipar*np4*neppM + ip4*neppM + ieppM]; // AOSOA[ipagM][ipar][ip4][ieppM] //fptype (*momenta)[npar][np4][neppM] = (fptype (*)[npar][np4][neppM]) momenta; // cast to multiD array pointer (AOSOA) //return momenta[ipagM][ipar][ip4][ieppM]; // this seems ~1-2% faster in eemumu C++? } -#else - // Return by value: it seems a tiny bit faster than returning a reference (both for scalar and vector), not clear why + +#ifndef __CUDACC__ + // Return a SIMD vector of fptype's for neppV events (for the given particle, 4-momentum component and event "V-page") + // Return the vector by value: strictly speaking, this is only unavoidable for neppM=neppV (both being powers of 2), the momentum neppM-AOSOA is reinterpreted in terms of neppV-vectors: + // it could also be returned by reference, but no performance degradation is observed when returning by value inline - fptype_sv pIparIp4Ipag( const fptype_sv* momenta, // input: momenta as AOSOA[npagM][npar][4][neppM] + fptype_sv pIparIp4Ipag( const fptype* momenta, // input: momenta as AOSOA[npagM][npar][4][neppM] const int ipar, const int ip4, - const int ipagM ) + const int ipagV ) { - // NB: this assumes that neppV == neppM! - // NB: this is the same as "pIparIp4Ievt( momenta, ipar, ip4, ipagM )" for neppM==1 with neppV==1 - using mgOnGpu::np4; - using mgOnGpu::npar; - //printf( "%f\n", momenta[ipagM*npar*np4 + ipar*np4 + ip4] ); - return momenta[ipagM*npar*np4 + ipar*np4 + ip4]; // AOSOA[ipagM][ipar][ip4][ieppM] + const int ievt0 = ipagV*neppV; // virtual event V-page ipagV contains neppV events [ievt0...ievt0+neppV-1] +#ifdef MGONGPU_CPPSIMD + constexpr int neppM = mgOnGpu::neppM; // AOSOA layout: constant at compile-time + // Use c++17 "if constexpr": compile-time branching + if constexpr ( ( neppM >= neppV ) && ( neppM%neppV == 0 ) ) + { + constexpr bool useReinterpretCastIfPossible = true; // DEFAULT + //constexpr bool useReinterpretCastIfPossible = false; // FOR PERFORMANCE TESTS + constexpr bool skipAlignmentCheck = true; // DEFAULT (MAY SEGFAULT, NEEDS A SANITY CHECK ELSEWHERE!) + //constexpr bool skipAlignmentCheck = false; // SLOWER BUT SAFER + if constexpr ( useReinterpretCastIfPossible && skipAlignmentCheck ) + { + //static bool first=true; if( first ){ std::cout << "WARNING! skip alignment check" << std::endl; first=false; } // SLOWS DOWN... + // Fastest (4.92E6 in eemumu 512y) + // This requires alignment for momenta - segmentation fault otherwise! + return *reinterpret_cast( &( pIparIp4Ievt( momenta, ipar, ip4, ievt0 ) ) ); + } + else if ( useReinterpretCastIfPossible && ( (size_t)(momenta) % mgOnGpu::cppAlign == 0 ) ) + { + //static bool first=true; if( first ){ std::cout << "WARNING! alignment ok, use reinterpret cast" << std::endl; first=false; } // SLOWS DOWN... + // A bit (6%) slower (4.62E6 in eemumu 512y) because of the alignment check + // This explicitly checks alignment for momenta to avoid segmentation faults + return *reinterpret_cast( &( pIparIp4Ievt( momenta, ipar, ip4, ievt0 ) ) ); + } + else + { + //static bool first=true; if( first ){ std::cout << "WARNING! AOSOA but no reinterpret cast" << std::endl; first=false; } // SLOWS DOWN... + // A bit (2%) slower (4.86E6 in eemumu 512y) + // This does not require alignment for momenta, but it requires AOSOA with neppM>=neppV and neppM%neppV==0 + const fptype* out0 = &( pIparIp4Ievt( momenta, ipar, ip4, ievt0) ); +#if MGONGPU_CPPSIMD == 2 + return fptype_v{ *( out0 ), + *( out0+1 ) }; +#elif MGONGPU_CPPSIMD == 4 + return fptype_v{ *( out0 ), + *( out0+1 ), + *( out0+2 ), + *( out0+3 ) }; +#elif MGONGPU_CPPSIMD == 8 + return fptype_v{ *( out0 ), + *( out0+1 ), + *( out0+2 ), + *( out0+3 ), + *( out0+4 ), + *( out0+5 ), + *( out0+6 ), + *( out0+7 ) }; +#elif MGONGPU_CPPSIMD == 16 + return fptype_v{ *( out0 ), + *( out0+1 ), + *( out0+2 ), + *( out0+3 ), + *( out0+4 ), + *( out0+5 ), + *( out0+6 ), + *( out0+7 ), + *( out0+8 ), + *( out0+9 ), + *( out0+10 ), + *( out0+11 ), + *( out0+12 ), + *( out0+13 ), + *( out0+14 ), + *( out0+15 ) }; +#else +#warning Internal error? This code should never be reached! + // A bit (2%) slower (4.86E6 in eemumu 512y) + // This does not require alignment for momenta, but it requires AOSOA with neppM>=neppV and neppM%neppV==0 + fptype_v out; + for ( int ieppV=0; ieppV=neppV and neppM%neppV==0 (e.g. can be used with AOS neppM==1) +#if MGONGPU_CPPSIMD == 2 + return fptype_v{ pIparIp4Ievt( momenta, ipar, ip4, ievt0 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+1 ) }; +#elif MGONGPU_CPPSIMD == 4 + return fptype_v{ pIparIp4Ievt( momenta, ipar, ip4, ievt0 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+1 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+2 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+3 ) }; +#elif MGONGPU_CPPSIMD == 8 + return fptype_v{ pIparIp4Ievt( momenta, ipar, ip4, ievt0 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+1 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+2 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+3 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+4 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+5 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+6 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+7 ) }; +#elif MGONGPU_CPPSIMD == 16 + return fptype_v{ pIparIp4Ievt( momenta, ipar, ip4, ievt0 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+1 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+2 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+3 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+4 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+5 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+6 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+7 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+8 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+9 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+10 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+11 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+12 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+13 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+14 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+15 ) }; +#else +#warning Internal error? This code should never be reached! + // Much much (40%) slower (3.02E6 in eemumu 512y) + fptype_v out; + for ( int ieppV=0; ieppV 0) __device__ - void ipzxxx( const fptype_sv* momenta, + void ipzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -247,7 +368,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) __device__ - void imzxxx( const fptype_sv* momenta, + void imzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -296,7 +417,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PT > 0) __device__ - void ixzxxx( const fptype_sv* momenta, + void ixzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -357,7 +478,7 @@ namespace MG5_sm // Compute the output wavefunction vc[6] from the input momenta[npar*4*nevt] __device__ - void vxxxxx( const fptype_sv* momenta, + void vxxxxx( const fptype* momenta, const fptype vmass, // input: vector boson mass const int nhel, // input: -1, 0 (only if vmass!=0) or +1 (helicity of vector boson) const int nsv, // input: +1 (final) or -1 (initial) @@ -493,7 +614,7 @@ namespace MG5_sm // Compute the output wavefunction sc[3] from the input momenta[npar*4*nevt] __device__ - void sxxxxx( const fptype_sv* momenta, + void sxxxxx( const fptype* momenta, const fptype, // WARNING: input "smass" unused (missing in Fortran) - scalar boson mass const int, // WARNING: input "nhel" unused (missing in Fortran) - scalar has no helicity! const int nss, // input: +1 (final) or -1 (initial) @@ -534,7 +655,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] __device__ - void oxxxxx( const fptype_sv* momenta, + void oxxxxx( const fptype* momenta, const fptype fmass, // input: fermion mass const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -671,7 +792,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) __device__ - void opzxxx( const fptype_sv* momenta, + void opzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -720,7 +841,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) __device__ - void omzxxx( const fptype_sv* momenta, + void omzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -772,7 +893,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PT > 0) __device__ - void oxzxxx( const fptype_sv* momenta, + void oxzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) diff --git a/epochX/cudacpp/ee_mumu/src/HelAmps_sm.h b/epochX/cudacpp/ee_mumu/src/HelAmps_sm.h index 877ea1505d..7e2a127c98 100644 --- a/epochX/cudacpp/ee_mumu/src/HelAmps_sm.h +++ b/epochX/cudacpp/ee_mumu/src/HelAmps_sm.h @@ -28,7 +28,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] __device__ INLINE - void ixxxxx( const fptype_sv* momenta, + void ixxxxx( const fptype* momenta, const fptype fmass, // input: fermion mass const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -43,7 +43,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) __device__ INLINE - void ipzxxx( const fptype_sv* momenta, + void ipzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -58,7 +58,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) __device__ INLINE - void imzxxx( const fptype_sv* momenta, + void imzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -73,7 +73,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PT > 0) __device__ INLINE - void ixzxxx( const fptype_sv* momenta, + void ixzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -87,7 +87,7 @@ namespace MG5_sm // Compute the output wavefunction vc[6] from the input momenta[npar*4*nevt] __device__ INLINE - void vxxxxx( const fptype_sv* momenta, + void vxxxxx( const fptype* momenta, const fptype vmass, // input: vector boson mass const int nhel, // input: -1, 0 (only if vmass!=0) or +1 (helicity of vector boson) const int nsv, // input: +1 (final) or -1 (initial) @@ -101,7 +101,7 @@ namespace MG5_sm // Compute the output wavefunction sc[3] from the input momenta[npar*4*nevt] __device__ INLINE - void sxxxxx( const fptype_sv* momenta, + void sxxxxx( const fptype* momenta, const fptype, // WARNING: input "smass" unused (missing in Fortran) - scalar boson mass const int, // WARNING: input "nhel" unused (missing in Fortran) - scalar has no helicity! const int nss, // input: +1 (final) or -1 (initial) @@ -115,7 +115,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] __device__ INLINE - void oxxxxx( const fptype_sv* momenta, + void oxxxxx( const fptype* momenta, const fptype fmass, // input: fermion mass const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -130,7 +130,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) __device__ INLINE - void opzxxx( const fptype_sv* momenta, + void opzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -145,7 +145,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) __device__ INLINE - void omzxxx( const fptype_sv* momenta, + void omzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -159,7 +159,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] __device__ INLINE - void oxzxxx( const fptype_sv* momenta, + void oxzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) diff --git a/epochX/cudacpp/ee_mumu/src/mgOnGpuConfig.h b/epochX/cudacpp/ee_mumu/src/mgOnGpuConfig.h index 39b12d3515..3b639644a3 100644 --- a/epochX/cudacpp/ee_mumu/src/mgOnGpuConfig.h +++ b/epochX/cudacpp/ee_mumu/src/mgOnGpuConfig.h @@ -96,47 +96,79 @@ namespace mgOnGpu //const int ntpbMAX = 256; // AV Apr2021: why had I set this to 256? const int ntpbMAX = 1024; // NB: 512 is ok, but 1024 does fail with "too many resources requested for launch" - // Vector sizes for AOSOA memory layouts (GPU coalesced memory access, CPU SIMD vectorization) - // (these are all best kept as a compile-time constants: see issue #23) + // Alignment requirement for using reinterpret_cast with SIMD vectorized code + // (using reinterpret_cast with non aligned memory may lead to segmentation faults!) +#ifndef __CUDACC__ + constexpr int cppAlign = 64; // alignment requirement for SIMD vectorization (64-byte i.e. 512-bit) +#endif - // Number of Events Per Page in the momenta AOSOA memory layout + // C++ SIMD vectorization width (this will be used to set neppV) #ifdef __CUDACC__ #undef MGONGPU_CPPSIMD - // ----------------------------------------------------------------------------------------- - // --- GPUs: neppM must be a power of 2 times the number of fptype's in a 32-byte cacheline - // --- This is relevant to ensure coalesced access to momenta in global memory - // --- Note that neppR is hardcoded and may differ from neppM and neppV on some platforms - // ----------------------------------------------------------------------------------------- - //const int neppM = 64/sizeof(fptype); // 2x 32-byte GPU cache lines: 8 (DOUBLE) or 16 (FLOAT) - const int neppM = 32/sizeof(fptype); // (DEFAULT) 32-byte GPU cache line: 4 (DOUBLE) or 8 (FLOAT) - //const int neppM = 1; // *** NB: this is equivalent to AOS *** - //const int neppM = 32; // older default #else - // ----------------------------------------------------------------------------------------- - // --- CPUs: neppM must be exactly equal to the number of fptype's in a vector register - // --- [DEFAULT is 256-width AVX512 aka "512y" on CPUs, 32-byte as GPUs, faster than AVX2] - // --- The logic of the code requires the size neppV of fptype_v to be equal to neppM - // --- Note that neppR is hardcoded and may differ from neppM and neppV on some platforms - // ----------------------------------------------------------------------------------------- #if defined __AVX512VL__ -#define MGONGPU_CPPSIMD 1 #ifdef MGONGPU_PVW512 - const int neppM = 64/sizeof(fptype); // "512z" AVX512 with 512 width (512-bit ie 64-byte): 8 (DOUBLE) or 16 (FLOAT) + // "512z" AVX512 with 512 width (512-bit ie 64-byte): 8 (DOUBLE) or 16 (FLOAT) +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 8 #else - const int neppM = 32/sizeof(fptype); // "512y" AVX512 with 256 width (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [gcc DEFAULT] +#define MGONGPU_CPPSIMD 16 +#endif +#else + // "512y" AVX512 with 256 width (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [gcc DEFAULT] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 4 +#else +#define MGONGPU_CPPSIMD 8 +#endif #endif #elif defined __AVX2__ -#define MGONGPU_CPPSIMD 1 - const int neppM = 32/sizeof(fptype); // "avx2" AVX2 (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [clang DEFAULT] + // "avx2" AVX2 (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [clang DEFAULT] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 4 +#else +#define MGONGPU_CPPSIMD 8 +#endif #elif defined __SSE4_2__ -#define MGONGPU_CPPSIMD 1 - const int neppM = 16/sizeof(fptype); // "sse4" SSE4.2 (128-bit ie 16-byte): 2 (DOUBLE) or 4 (FLOAT) + // "sse4" SSE4.2 (128-bit ie 16-byte): 2 (DOUBLE) or 4 (FLOAT) +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 2 #else +#define MGONGPU_CPPSIMD 4 +#endif +#else + // "none" i.e. no SIMD (*** NB: this is equivalent to AOS ***) #undef MGONGPU_CPPSIMD - const int neppM = 1; // "none" i.e. no SIMD (*** NB: this is equivalent to AOS ***) #endif #endif + // Number of Events Per Page in the momenta AOSOA memory layout +#ifdef __CUDACC__ + // ----------------------------------------------------------------------------------------------- + // --- GPUs: neppM is best set to a power of 2 times the number of fptype's in a 32-byte cacheline + // --- This is relevant to ensure coalesced access to momenta in global memory + // --- Note that neppR is hardcoded and may differ from neppM and neppV on some platforms + // ----------------------------------------------------------------------------------------------- + //const int neppM = 64/sizeof(fptype); // 2x 32-byte GPU cache lines (512 bits): 8 (DOUBLE) or 16 (FLOAT) + const int neppM = 32/sizeof(fptype); // (DEFAULT) 32-byte GPU cache line (256 bits): 4 (DOUBLE) or 8 (FLOAT) + //const int neppM = 1; // *** NB: this is equivalent to AOS *** +#else + // ----------------------------------------------------------------------------------------------- + // --- CPUs: neppM is best set equal to the number of fptype's (neppV) in a vector register + // --- This is relevant to ensure faster access to momenta from C++ memory cache lines + // --- However, neppM is now decoupled from neppV (issue #176) and can be separately hardcoded + // --- In practice, neppR, neppM and neppV could now (in principle) all be different + // ----------------------------------------------------------------------------------------------- +#ifdef MGONGPU_CPPSIMD + const int neppM = MGONGPU_CPPSIMD; // (DEFAULT) neppM=neppV for optimal performance +#else + const int neppM = 1; // (DEFAULT) neppM=neppV for optimal performance (NB: this is equivalent to AOS) +#endif + //const int neppM = 64/sizeof(fptype); // maximum CPU vector width (512 bits): 8 (DOUBLE) or 16 (FLOAT) + //const int neppM = 32/sizeof(fptype); // lower CPU vector width (256 bits): 4 (DOUBLE) or 8 (FLOAT) + //const int neppM = 1; // *** NB: this is equivalent to AOS *** +#endif + // Number of Events Per Page in the random number AOSOA memory layout // *** NB Different values of neppR lead to different physics results: the *** // *** same 1d array is generated, but it is interpreted in different ways *** diff --git a/epochX/cudacpp/ee_mumu/src/mgOnGpuVectors.h b/epochX/cudacpp/ee_mumu/src/mgOnGpuVectors.h index 9e8b34988f..4e975b2ca1 100644 --- a/epochX/cudacpp/ee_mumu/src/mgOnGpuVectors.h +++ b/epochX/cudacpp/ee_mumu/src/mgOnGpuVectors.h @@ -13,9 +13,13 @@ namespace mgOnGpu { + #ifdef MGONGPU_CPPSIMD - const int neppV = neppM; + const int neppV = MGONGPU_CPPSIMD; + + // SANITY CHECK: cppAlign must be a multiple of neppV * sizeof(fptype) + static_assert( mgOnGpu::cppAlign % ( neppV * sizeof(fptype) ) == 0 ); // --- Type definition (using vector compiler extensions: need -march=...) #ifdef __clang__ // https://clang.llvm.org/docs/LanguageExtensions.html#vectors-and-extended-vectors @@ -96,7 +100,7 @@ namespace mgOnGpu #else // MGONGPU_CPPSIMD not defined - const int neppV = 1; // Note: also neppM is equal to 1 + const int neppV = 1; #endif } diff --git a/epochX/cudacpp/ee_mumu/src/rambo.cc b/epochX/cudacpp/ee_mumu/src/rambo.cc index 0a85ef30be..220e8d2b87 100644 --- a/epochX/cudacpp/ee_mumu/src/rambo.cc +++ b/epochX/cudacpp/ee_mumu/src/rambo.cc @@ -2,7 +2,7 @@ #include "mgOnGpuConfig.h" #include "mgOnGpuTypes.h" -#include "mgOnGpuVectors.h" +//#include "mgOnGpuVectors.h" #include #include @@ -22,7 +22,7 @@ namespace rambo2toNm0 // [NB: the output buffer includes both initial and final momenta, but only initial momenta are filled in] __global__ void getMomentaInitial( const fptype energy, // input: energy - fptype_sv momenta1d[] // output: momenta as AOSOA[npagM][npar][4][neppM] + fptype momenta1d[] // output: momenta as AOSOA[npagM][npar][4][neppM] #ifndef __CUDACC__ , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) #endif @@ -63,7 +63,7 @@ namespace rambo2toNm0 __global__ void getMomentaFinal( const fptype energy, // input: energy const fptype rnarray1d[], // input: random numbers in [0,1] as AOSOA[npagR][nparf][4][neppR] - fptype_sv momenta1d[], // output: momenta as AOSOA[npagM][npar][4][neppM] + fptype momenta1d[], // output: momenta as AOSOA[npagM][npar][4][neppM] fptype wgts[] // output: weights[nevt] #ifndef __CUDACC__ , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) diff --git a/epochX/cudacpp/ee_mumu/src/rambo.h b/epochX/cudacpp/ee_mumu/src/rambo.h index 030526759f..9d9e876fe3 100644 --- a/epochX/cudacpp/ee_mumu/src/rambo.h +++ b/epochX/cudacpp/ee_mumu/src/rambo.h @@ -3,7 +3,7 @@ #include "mgOnGpuConfig.h" #include "mgOnGpuTypes.h" -#include "mgOnGpuVectors.h" +//#include "mgOnGpuVectors.h" #include @@ -47,7 +47,7 @@ namespace rambo2toNm0 // [NB: the output buffer includes both initial and final momenta, but only initial momenta are filled in] __global__ void getMomentaInitial( const fptype energy, // input: energy - fptype_sv momenta1d[] // output: momenta as AOSOA[npagM][npar][4][neppM] + fptype momenta1d[] // output: momenta as AOSOA[npagM][npar][4][neppM] #ifndef __CUDACC__ , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) #endif @@ -60,7 +60,7 @@ namespace rambo2toNm0 __global__ void getMomentaFinal( const fptype energy, // input: energy const fptype rnarray1d[], // input: random numbers in [0,1] as AOSOA[npagR][nparf][4][neppR] - fptype_sv momenta1d[], // output: momenta as AOSOA[npagM][npar][4][neppM] + fptype momenta1d[], // output: momenta as AOSOA[npagM][npar][4][neppM] fptype wgts[] // output: weights[nevt] #ifndef __CUDACC__ , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) From 7a879bd552d9936b0e2ba8c060a1cca7ef5956c8 Mon Sep 17 00:00:00 2001 From: Andrea Valassi Date: Wed, 1 Dec 2021 08:50:49 +0100 Subject: [PATCH 05/34] [klas5X] performance with the last commit - removed neppV from API Tiny performance drop probably because of removed vectorization in allMEs initialization? --- .../log_eemumu_manu_d_inl0.txt | 106 +++++++++--------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_d_inl0.txt b/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_d_inl0.txt index 10f68d04b2..34dde48eb0 100644 --- a/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_d_inl0.txt +++ b/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_d_inl0.txt @@ -1,5 +1,5 @@ -/afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum +/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum OMPFLAGS= AVX=none FPTYPE=d @@ -40,102 +40,102 @@ RNDGEN=curdev Building in BUILDDIR=build.512z_d_inl0 for tag=512z_d_inl0_curdev (USEBUILDDIR is set = 1) make: Nothing to be done for `all.512z_d_inl0_curdev'. -DATE: 2021-10-30_18:37:47 +DATE: 2021-11-30_18:26:43 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl0/gcheck.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl0/gcheck.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 6.529901e+08 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.111698e+09 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 6.429697e+08 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.106342e+09 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 1.023786 sec - 1,013,108,369 cycles:u # 0.831 GHz - 2,010,561,954 instructions:u # 1.98 insn per cycle - 1.327096906 seconds time elapsed +TOTAL : 0.707901 sec + 373,331,879 cycles:u # 0.396 GHz + 693,060,121 instructions:u # 1.86 insn per cycle + 1.006807629 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 130 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl0/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl0/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.318021e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.318021e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.340565e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.340565e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 7.161641 sec - 18,905,744,448 cycles:u # 2.637 GHz - 48,224,166,376 instructions:u # 2.55 insn per cycle - 7.175443997 seconds time elapsed +TOTAL : 7.024663 sec + 18,630,578,672 cycles:u # 2.651 GHz + 48,224,156,791 instructions:u # 2.59 insn per cycle + 7.032416618 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 621) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl0/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_d_inl0/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_d_inl0/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[2] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 2.534746e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 2.534746e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 2.566794e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 2.566794e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 4.889071 sec - 12,819,823,040 cycles:u # 2.618 GHz - 29,912,703,447 instructions:u # 2.33 insn per cycle - 4.903117790 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 3327) (avx2: 0) (512y: 0) (512z: 0) +TOTAL : 4.832176 sec + 12,735,236,411 cycles:u # 2.634 GHz + 29,850,563,842 instructions:u # 2.34 insn per cycle + 4.840327392 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 3297) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_d_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_d_inl0/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_d_inl0/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_d_inl0/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 4.500007e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.500007e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.559694e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.559694e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 3.734716 sec - 9,140,145,635 cycles:u # 2.440 GHz - 16,678,502,530 instructions:u # 1.82 insn per cycle - 3.748869006 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2806) (512y: 0) (512z: 0) +TOTAL : 3.662016 sec + 9,085,677,498 cycles:u # 2.478 GHz + 16,644,667,698 instructions:u # 1.83 insn per cycle + 3.669537861 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2831) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_d_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_d_inl0/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_d_inl0/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_d_inl0/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 4.976394e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.976394e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.934023e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.934023e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 3.600360 sec - 8,900,342,467 cycles:u # 2.467 GHz - 16,565,258,941 instructions:u # 1.86 insn per cycle - 3.613976249 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2703) (512y: 59) (512z: 0) +TOTAL : 3.571025 sec + 8,891,334,309 cycles:u # 2.487 GHz + 16,531,423,280 instructions:u # 1.86 insn per cycle + 3.578914335 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2728) (512y: 52) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_d_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_d_inl0/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_d_inl0/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_d_inl0/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 3.666462e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.666462e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 3.817528e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.817528e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 4.074651 sec - 8,790,835,737 cycles:u # 2.152 GHz - 13,329,837,352 instructions:u # 1.52 insn per cycle - 4.088934994 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1251) (512y: 68) (512z: 2163) +TOTAL : 3.968037 sec + 8,660,592,511 cycles:u # 2.180 GHz + 13,280,272,764 instructions:u # 1.53 insn per cycle + 3.975661225 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1319) (512y: 66) (512z: 2147) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_d_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_d_inl0/runTest.exe [ PASSED ] 3 tests. ========================================================================= From a2f3388badcba7ba2d55b7d6d42801ff587d4632 Mon Sep 17 00:00:00 2001 From: Andrea Valassi Date: Wed, 1 Dec 2021 09:07:58 +0100 Subject: [PATCH 06/34] [klas5X] try to improve performance in allMEs init to 0 - no change, will revert --- .../P1_Sigma_sm_epem_mupmum/CPPProcess.cc | 18 ++++- .../log_eemumu_manu_d_inl0.txt | 74 +++++++++---------- 2 files changed, 53 insertions(+), 39 deletions(-) diff --git a/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc b/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc index d49bf181ce..7283f18eaf 100644 --- a/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc +++ b/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc @@ -495,11 +495,25 @@ namespace Proc #ifdef __CUDACC__ allMEs[ievt] = 0; #else +#ifdef MGONGPU_CPPSIMD + const bool isAligned_allMEs = ( (size_t)(allMEs) % mgOnGpu::cppAlign == 0 ); // require SIMD-friendly alignment by at least neppV*sizeof(fptype) +#endif const int npagV = nevt/neppV; for ( int ipagV = 0; ipagV < npagV; ++ipagV ) { - for ( int ieppV=0; ieppV( &( allMEs[ipagV*neppV] ) ) = fptype_sv{0}; // all zeros + } + else + { + for ( int ieppV=0; ieppV Date: Wed, 1 Dec 2021 09:08:30 +0100 Subject: [PATCH 07/34] Revert "[klas5X] try to improve performance in allMEs init to 0 - no change, will revert" This reverts commit a2f3388badcba7ba2d55b7d6d42801ff587d4632. --- .../P1_Sigma_sm_epem_mupmum/CPPProcess.cc | 18 +---- .../log_eemumu_manu_d_inl0.txt | 74 +++++++++---------- 2 files changed, 39 insertions(+), 53 deletions(-) diff --git a/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc b/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc index 7283f18eaf..d49bf181ce 100644 --- a/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc +++ b/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc @@ -495,25 +495,11 @@ namespace Proc #ifdef __CUDACC__ allMEs[ievt] = 0; #else -#ifdef MGONGPU_CPPSIMD - const bool isAligned_allMEs = ( (size_t)(allMEs) % mgOnGpu::cppAlign == 0 ); // require SIMD-friendly alignment by at least neppV*sizeof(fptype) -#endif const int npagV = nevt/neppV; for ( int ipagV = 0; ipagV < npagV; ++ipagV ) { -#ifdef MGONGPU_CPPSIMD - if ( isAligned_allMEs ) - { - *reinterpret_cast( &( allMEs[ipagV*neppV] ) ) = fptype_sv{0}; // all zeros - } - else - { - for ( int ieppV=0; ieppV Date: Wed, 1 Dec 2021 09:17:01 +0100 Subject: [PATCH 08/34] [klas5X] try to improve performance in allMEs "/= denominators" - very tiny change, will revert (note: this time, unlike the "=0" changes, I see small changes in the symbols used, but not sure they are good) --- .../P1_Sigma_sm_epem_mupmum/CPPProcess.cc | 18 +++- .../log_eemumu_manu_d_inl0.txt | 82 +++++++++---------- 2 files changed, 57 insertions(+), 43 deletions(-) diff --git a/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc b/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc index d49bf181ce..acd5b8bb44 100644 --- a/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc +++ b/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc @@ -525,10 +525,24 @@ namespace Proc #ifdef __CUDACC__ allMEs[ievt] /= denominators; #else +#ifdef MGONGPU_CPPSIMD + const bool isAligned_allMEs = ( (size_t)(allMEs) % mgOnGpu::cppAlign == 0 ); // require SIMD-friendly alignment by at least neppV*sizeof(fptype) +#endif for ( int ipagV = 0; ipagV < npagV; ++ipagV ) { - for ( int ieppV=0; ieppV( &( allMEs[ipagV*neppV] ) ) /= denominators; + } + else + { + for ( int ieppV=0; ieppV Date: Wed, 1 Dec 2021 09:18:08 +0100 Subject: [PATCH 09/34] Revert "[klas5X] try to improve performance in allMEs "/= denominators" - very tiny change, will revert" This reverts commit a26e057805c9e461674eb0fb91dca03f8f62ff2d. --- .../P1_Sigma_sm_epem_mupmum/CPPProcess.cc | 18 +--- .../log_eemumu_manu_d_inl0.txt | 82 +++++++++---------- 2 files changed, 43 insertions(+), 57 deletions(-) diff --git a/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc b/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc index acd5b8bb44..d49bf181ce 100644 --- a/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc +++ b/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc @@ -525,24 +525,10 @@ namespace Proc #ifdef __CUDACC__ allMEs[ievt] /= denominators; #else -#ifdef MGONGPU_CPPSIMD - const bool isAligned_allMEs = ( (size_t)(allMEs) % mgOnGpu::cppAlign == 0 ); // require SIMD-friendly alignment by at least neppV*sizeof(fptype) -#endif for ( int ipagV = 0; ipagV < npagV; ++ipagV ) { -#ifdef MGONGPU_CPPSIMD - if ( isAligned_allMEs ) - { - *reinterpret_cast( &( allMEs[ipagV*neppV] ) ) /= denominators; - } - else - { - for ( int ieppV=0; ieppV Date: Wed, 1 Dec 2021 09:59:57 +0100 Subject: [PATCH 10/34] [klas5X] backport to CODEGEN the latest klas5X changes from eemumu manual - eemumu auto is now identical --- .../aloha/template_files/gpu/helas.cu | 183 ++++++++++++++---- .../aloha/template_files/gpu/helas.h | 20 +- .../iolibs/template_files/gpu/Memory.h | 19 +- .../iolibs/template_files/gpu/check_sa.cc | 35 +--- .../iolibs/template_files/gpu/mgOnGpuConfig.h | 84 +++++--- .../template_files/gpu/mgOnGpuVectors.h | 16 +- .../gpu/process_function_definitions.inc | 43 ++-- .../iolibs/template_files/gpu/process_h.inc | 14 +- .../template_files/gpu/process_matrix.inc | 20 +- .../gpu/process_sigmaKin_function.inc | 6 +- .../iolibs/template_files/gpu/rambo.cc | 6 +- .../iolibs/template_files/gpu/rambo.h | 6 +- .../iolibs/template_files/gpu/runTest.cc | 36 ++-- .../iolibs/template_files/gpu/testxxx.cc | 8 +- .../CUDACPP_SA_OUTPUT/model_handling.py | 13 +- 15 files changed, 335 insertions(+), 174 deletions(-) diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/aloha/template_files/gpu/helas.cu b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/aloha/template_files/gpu/helas.cu index 2a87781920..08f5b30a0a 100644 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/aloha/template_files/gpu/helas.cu +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/aloha/template_files/gpu/helas.cu @@ -1,7 +1,7 @@ //-------------------------------------------------------------------------- -#ifdef __CUDACC__ - // Return by reference + // Decode momentum AOSOA: compute address of fptype for the given particle, 4-momentum component and event + // Return the fptype by reference (equivalent to returning its memory address) __device__ inline const fptype& pIparIp4Ievt( const fptype* momenta, // input: momenta as AOSOA[npagM][npar][4][neppM] const int ipar, @@ -10,38 +10,149 @@ { using mgOnGpu::np4; using mgOnGpu::npar; - const int neppM = mgOnGpu::neppM; // AOSOA layout: constant at compile-time - const int ipagM = ievt/neppM; // #eventpage in this iteration - const int ieppM = ievt%neppM; // #event in the current eventpage in this iteration - //printf( "%f\n", momenta[ipagM*npar*np4*neppM + ipar*np4*neppM + ip4*neppM + ieppM] ); + constexpr int neppM = mgOnGpu::neppM; // AOSOA layout: constant at compile-time + const int ipagM = ievt/neppM; // #event "M-page" + const int ieppM = ievt%neppM; // #event in the current event M-page + //printf( "%2d %2d %8d %8.3f\n", ipar, ip4, ievt, momenta[ipagM*npar*np4*neppM + ipar*np4*neppM + ip4*neppM + ieppM] ); return momenta[ipagM*npar*np4*neppM + ipar*np4*neppM + ip4*neppM + ieppM]; // AOSOA[ipagM][ipar][ip4][ieppM] //fptype (*momenta)[npar][np4][neppM] = (fptype (*)[npar][np4][neppM]) momenta; // cast to multiD array pointer (AOSOA) //return momenta[ipagM][ipar][ip4][ieppM]; // this seems ~1-2% faster in eemumu C++? } -#else - // Return by value: it seems a tiny bit faster than returning a reference (both for scalar and vector), not clear why + +#ifndef __CUDACC__ + // Return a SIMD vector of fptype's for neppV events (for the given particle, 4-momentum component and event "V-page") + // Return the vector by value: strictly speaking, this is only unavoidable for neppM=neppV (both being powers of 2), the momentum neppM-AOSOA is reinterpreted in terms of neppV-vectors: + // it could also be returned by reference, but no performance degradation is observed when returning by value inline - fptype_sv pIparIp4Ipag( const fptype_sv* momenta, // input: momenta as AOSOA[npagM][npar][4][neppM] + fptype_sv pIparIp4Ipag( const fptype* momenta, // input: momenta as AOSOA[npagM][npar][4][neppM] const int ipar, const int ip4, - const int ipagM ) + const int ipagV ) { - /* - //#ifndef MGONGPU_CPPSIMD - // TEMPORARY during epochX step3! Eventually remove this section (start) - // TEMPORARY during epochX step3! THERE IS NO SIMD YET: HENCE ipagM=ievt - // NB: this is needed for neppM>1 while neppV==1 (no SIMD) in eemumu.auto - // NB: this remains valid for neppM==1 with neppV==1 in eemumu (scalar) - return pIparIp4Ievt( momenta, ipar, ip4, ipagM ); - // TEMPORARY during epochX step3! Eventually remove this section (end) - //#else - */ - // NB: this assumes that neppV == neppM! - // NB: this is the same as "pIparIp4Ievt( momenta, ipar, ip4, ipagM )" for neppM==1 with neppV==1 - using mgOnGpu::np4; - using mgOnGpu::npar; - //printf( "%f\n", momenta[ipagM*npar*np4 + ipar*np4 + ip4] ); - return momenta[ipagM*npar*np4 + ipar*np4 + ip4]; // AOSOA[ipagM][ipar][ip4][ieppM] + const int ievt0 = ipagV*neppV; // virtual event V-page ipagV contains neppV events [ievt0...ievt0+neppV-1] +#ifdef MGONGPU_CPPSIMD + constexpr int neppM = mgOnGpu::neppM; // AOSOA layout: constant at compile-time + // Use c++17 "if constexpr": compile-time branching + if constexpr ( ( neppM >= neppV ) && ( neppM%neppV == 0 ) ) + { + constexpr bool useReinterpretCastIfPossible = true; // DEFAULT + //constexpr bool useReinterpretCastIfPossible = false; // FOR PERFORMANCE TESTS + constexpr bool skipAlignmentCheck = true; // DEFAULT (MAY SEGFAULT, NEEDS A SANITY CHECK ELSEWHERE!) + //constexpr bool skipAlignmentCheck = false; // SLOWER BUT SAFER + if constexpr ( useReinterpretCastIfPossible && skipAlignmentCheck ) + { + //static bool first=true; if( first ){ std::cout << "WARNING! skip alignment check" << std::endl; first=false; } // SLOWS DOWN... + // Fastest (4.92E6 in eemumu 512y) + // This requires alignment for momenta - segmentation fault otherwise! + return *reinterpret_cast( &( pIparIp4Ievt( momenta, ipar, ip4, ievt0 ) ) ); + } + else if ( useReinterpretCastIfPossible && ( (size_t)(momenta) % mgOnGpu::cppAlign == 0 ) ) + { + //static bool first=true; if( first ){ std::cout << "WARNING! alignment ok, use reinterpret cast" << std::endl; first=false; } // SLOWS DOWN... + // A bit (6%) slower (4.62E6 in eemumu 512y) because of the alignment check + // This explicitly checks alignment for momenta to avoid segmentation faults + return *reinterpret_cast( &( pIparIp4Ievt( momenta, ipar, ip4, ievt0 ) ) ); + } + else + { + //static bool first=true; if( first ){ std::cout << "WARNING! AOSOA but no reinterpret cast" << std::endl; first=false; } // SLOWS DOWN... + // A bit (2%) slower (4.86E6 in eemumu 512y) + // This does not require alignment for momenta, but it requires AOSOA with neppM>=neppV and neppM%neppV==0 + const fptype* out0 = &( pIparIp4Ievt( momenta, ipar, ip4, ievt0) ); +#if MGONGPU_CPPSIMD == 2 + return fptype_v{ *( out0 ), + *( out0+1 ) }; +#elif MGONGPU_CPPSIMD == 4 + return fptype_v{ *( out0 ), + *( out0+1 ), + *( out0+2 ), + *( out0+3 ) }; +#elif MGONGPU_CPPSIMD == 8 + return fptype_v{ *( out0 ), + *( out0+1 ), + *( out0+2 ), + *( out0+3 ), + *( out0+4 ), + *( out0+5 ), + *( out0+6 ), + *( out0+7 ) }; +#elif MGONGPU_CPPSIMD == 16 + return fptype_v{ *( out0 ), + *( out0+1 ), + *( out0+2 ), + *( out0+3 ), + *( out0+4 ), + *( out0+5 ), + *( out0+6 ), + *( out0+7 ), + *( out0+8 ), + *( out0+9 ), + *( out0+10 ), + *( out0+11 ), + *( out0+12 ), + *( out0+13 ), + *( out0+14 ), + *( out0+15 ) }; +#else +#warning Internal error? This code should never be reached! + // A bit (2%) slower (4.86E6 in eemumu 512y) + // This does not require alignment for momenta, but it requires AOSOA with neppM>=neppV and neppM%neppV==0 + fptype_v out; + for ( int ieppV=0; ieppV=neppV and neppM%neppV==0 (e.g. can be used with AOS neppM==1) +#if MGONGPU_CPPSIMD == 2 + return fptype_v{ pIparIp4Ievt( momenta, ipar, ip4, ievt0 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+1 ) }; +#elif MGONGPU_CPPSIMD == 4 + return fptype_v{ pIparIp4Ievt( momenta, ipar, ip4, ievt0 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+1 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+2 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+3 ) }; +#elif MGONGPU_CPPSIMD == 8 + return fptype_v{ pIparIp4Ievt( momenta, ipar, ip4, ievt0 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+1 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+2 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+3 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+4 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+5 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+6 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+7 ) }; +#elif MGONGPU_CPPSIMD == 16 + return fptype_v{ pIparIp4Ievt( momenta, ipar, ip4, ievt0 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+1 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+2 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+3 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+4 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+5 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+6 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+7 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+8 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+9 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+10 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+11 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+12 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+13 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+14 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+15 ) }; +#else +#warning Internal error? This code should never be reached! + // Much much (40%) slower (3.02E6 in eemumu 512y) + fptype_v out; + for ( int ieppV=0; ieppV 0) __device__ - void ipzxxx( const fptype_sv* momenta, + void ipzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -234,7 +345,7 @@ // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) __device__ - void imzxxx( const fptype_sv* momenta, + void imzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -283,7 +394,7 @@ // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PT > 0) __device__ - void ixzxxx( const fptype_sv* momenta, + void ixzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -344,7 +455,7 @@ // Compute the output wavefunction vc[6] from the input momenta[npar*4*nevt] __device__ - void vxxxxx( const fptype_sv* momenta, + void vxxxxx( const fptype* momenta, const fptype vmass, // input: vector boson mass const int nhel, // input: -1, 0 (only if vmass!=0) or +1 (helicity of vector boson) const int nsv, // input: +1 (final) or -1 (initial) @@ -480,7 +591,7 @@ // Compute the output wavefunction sc[3] from the input momenta[npar*4*nevt] __device__ - void sxxxxx( const fptype_sv* momenta, + void sxxxxx( const fptype* momenta, const fptype, // WARNING: input "smass" unused (missing in Fortran) - scalar boson mass const int, // WARNING: input "nhel" unused (missing in Fortran) - scalar has no helicity! const int nss, // input: +1 (final) or -1 (initial) @@ -521,7 +632,7 @@ // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] __device__ - void oxxxxx( const fptype_sv* momenta, + void oxxxxx( const fptype* momenta, const fptype fmass, // input: fermion mass const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -658,7 +769,7 @@ // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) __device__ - void opzxxx( const fptype_sv* momenta, + void opzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -707,7 +818,7 @@ // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) __device__ - void omzxxx( const fptype_sv* momenta, + void omzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -759,7 +870,7 @@ // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PT > 0) __device__ - void oxzxxx( const fptype_sv* momenta, + void oxzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/aloha/template_files/gpu/helas.h b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/aloha/template_files/gpu/helas.h index 320e5e4a5c..f730953d96 100644 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/aloha/template_files/gpu/helas.h +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/aloha/template_files/gpu/helas.h @@ -11,7 +11,7 @@ // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] __device__ INLINE - void ixxxxx( const fptype_sv* momenta, + void ixxxxx( const fptype* momenta, const fptype fmass, // input: fermion mass const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -26,7 +26,7 @@ // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) __device__ INLINE - void ipzxxx( const fptype_sv* momenta, + void ipzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -41,7 +41,7 @@ // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) __device__ INLINE - void imzxxx( const fptype_sv* momenta, + void imzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -56,7 +56,7 @@ // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PT > 0) __device__ INLINE - void ixzxxx( const fptype_sv* momenta, + void ixzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -70,7 +70,7 @@ // Compute the output wavefunction vc[6] from the input momenta[npar*4*nevt] __device__ INLINE - void vxxxxx( const fptype_sv* momenta, + void vxxxxx( const fptype* momenta, const fptype vmass, // input: vector boson mass const int nhel, // input: -1, 0 (only if vmass!=0) or +1 (helicity of vector boson) const int nsv, // input: +1 (final) or -1 (initial) @@ -84,7 +84,7 @@ // Compute the output wavefunction sc[3] from the input momenta[npar*4*nevt] __device__ INLINE - void sxxxxx( const fptype_sv* momenta, + void sxxxxx( const fptype* momenta, const fptype, // WARNING: input "smass" unused (missing in Fortran) - scalar boson mass const int, // WARNING: input "nhel" unused (missing in Fortran) - scalar has no helicity! const int nss, // input: +1 (final) or -1 (initial) @@ -98,7 +98,7 @@ // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] __device__ INLINE - void oxxxxx( const fptype_sv* momenta, + void oxxxxx( const fptype* momenta, const fptype fmass, // input: fermion mass const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -113,7 +113,7 @@ // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) __device__ INLINE - void opzxxx( const fptype_sv* momenta, + void opzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -128,7 +128,7 @@ // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) __device__ INLINE - void omzxxx( const fptype_sv* momenta, + void omzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -142,7 +142,7 @@ // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] __device__ INLINE - void oxzxxx( const fptype_sv* momenta, + void oxzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/Memory.h b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/Memory.h index d7d1c7d4da..a9c08d3dd0 100644 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/Memory.h +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/Memory.h @@ -1,8 +1,9 @@ /* - * memory.h + * Memory.h * * Created on: 19.11.2020 * Author: shageboeck + * Modified: avalassi (vectors, alignment) */ #ifndef MEMORY_H @@ -46,13 +47,25 @@ std::unique_ptr> hstMakeUnique(std::size_t N) { #else +template +struct CppHstDeleter { + void operator()(T* mem) { + ::operator delete( mem, std::align_val_t{ mgOnGpu::cppAlign } ); + } +}; + template inline -std::unique_ptr hstMakeUnique(std::size_t N) { return std::unique_ptr{ new T[N]() }; }; +std::unique_ptr> hstMakeUnique(std::size_t N) { + // See https://www.bfilipek.com/2019/08/newnew-align.html#requesting-different-alignment + return std::unique_ptr>{ new( std::align_val_t{ mgOnGpu::cppAlign } ) T[N]() }; +}; #ifdef MGONGPU_CPPSIMD template<> inline -std::unique_ptr hstMakeUnique(std::size_t N) { return std::unique_ptr{ new fptype_v[N/neppV]() }; }; +std::unique_ptr> hstMakeUnique(std::size_t N) { + return std::unique_ptr>{ new( std::align_val_t{ mgOnGpu::cppAlign } ) fptype_v[N/neppV]() }; +}; #endif diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/check_sa.cc b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/check_sa.cc index 46d080459e..231e473d95 100644 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/check_sa.cc +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/check_sa.cc @@ -316,19 +316,19 @@ int main(int argc, char **argv) const int nMEs = nevt; // FIXME: assume process.nprocesses == 1 (eventually: nMEs = nevt * nprocesses?) #if defined MGONGPU_CURAND_ONHOST or defined MGONGPU_COMMONRAND_ONHOST or not defined __CUDACC__ - auto hstRnarray = hstMakeUnique( nRnarray ); // AOSOA[npagR][nparf][np4][neppR] (NB: nevt=npagR*neppR) + auto hstRnarray = hstMakeUnique( nRnarray ); // AOSOA[npagR][nparf][np4][neppR] (NB: nevt=npagR*neppR) #endif - auto hstMomenta = hstMakeUnique( nMomenta ); // AOSOA[npagM][npar][np4][neppM] (NB: nevt=npagM*neppM) - auto hstIsGoodHel = hstMakeUnique( ncomb ); - auto hstWeights = hstMakeUnique( nWeights ); - auto hstMEs = hstMakeUnique( nMEs ); // AOSOA[npagM][neppM] (NB: nevt=npagM*neppM) + auto hstMomenta = hstMakeUnique( nMomenta ); // AOSOA[npagM][npar][np4][neppM] (NB: nevt=npagM*neppM) + auto hstIsGoodHel = hstMakeUnique( ncomb ); + auto hstWeights = hstMakeUnique( nWeights ); + auto hstMEs = hstMakeUnique( nMEs ); // ARRAY[nevt] #ifdef __CUDACC__ - auto devRnarray = devMakeUnique( nRnarray ); // AOSOA[npagR][nparf][np4][neppR] (NB: nevt=npagR*neppR) - auto devMomenta = devMakeUnique( nMomenta ); // AOSOA[npagM][npar][np4][neppM] (NB: nevt=npagM*neppM) - auto devIsGoodHel = devMakeUnique( ncomb ); - auto devWeights = devMakeUnique( nWeights ); - auto devMEs = devMakeUnique( nMEs ); // AOSOA[npagM][neppM] (NB: nevt=npagM*neppM) + auto devRnarray = devMakeUnique( nRnarray ); // AOSOA[npagR][nparf][np4][neppR] (NB: nevt=npagR*neppR) + auto devMomenta = devMakeUnique( nMomenta ); // AOSOA[npagM][npar][np4][neppM] (NB: nevt=npagM*neppM) + auto devIsGoodHel = devMakeUnique( ncomb ); + auto devWeights = devMakeUnique( nWeights ); + auto devMEs = devMakeUnique( nMEs ); // ARRAY[nevt] #if defined MGONGPU_CURAND_ONHOST or defined MGONGPU_COMMONRAND_ONHOST const int nbytesRnarray = nRnarray * sizeof(fptype); @@ -553,37 +553,22 @@ int main(int argc, char **argv) // NB: 'setw' affects only the next field (of any type) std::cout << std::scientific // fixed format: affects all floats (default precision: 6) << std::setw(4) << ipar + 1 -#ifndef MGONGPU_CPPSIMD << std::setw(14) << hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + 0*neppM + ieppM] // AOSOA[ipagM][ipar][0][ieppM] << std::setw(14) << hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + 1*neppM + ieppM] // AOSOA[ipagM][ipar][1][ieppM] << std::setw(14) << hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + 2*neppM + ieppM] // AOSOA[ipagM][ipar][2][ieppM] << std::setw(14) << hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + 3*neppM + ieppM] // AOSOA[ipagM][ipar][3][ieppM] -#else - << std::setw(14) << hstMomenta[ipagM*npar*np4 + ipar*np4 + 0][ieppM] // AOSOA[ipagM][ipar][0][ieppM] - << std::setw(14) << hstMomenta[ipagM*npar*np4 + ipar*np4 + 1][ieppM] // AOSOA[ipagM][ipar][1][ieppM] - << std::setw(14) << hstMomenta[ipagM*npar*np4 + ipar*np4 + 2][ieppM] // AOSOA[ipagM][ipar][2][ieppM] - << std::setw(14) << hstMomenta[ipagM*npar*np4 + ipar*np4 + 3][ieppM] // AOSOA[ipagM][ipar][3][ieppM] -#endif << std::endl << std::defaultfloat; // default format: affects all floats } std::cout << std::string(SEP79, '-') << std::endl; // Display matrix elements std::cout << " Matrix element = " -#ifndef MGONGPU_CPPSIMD << hstMEs[ievt] -#else - << hstMEs[ievt/neppM][ievt%neppM] -#endif << " GeV^" << meGeVexponent << std::endl; // FIXME: assume process.nprocesses == 1 std::cout << std::string(SEP79, '-') << std::endl; } // Fill the arrays with ALL MEs and weights -#ifndef MGONGPU_CPPSIMD matrixelementALL[iiter*nevt + ievt] = hstMEs[ievt]; // FIXME: assume process.nprocesses == 1 -#else - matrixelementALL[iiter*nevt + ievt] = hstMEs[ievt/neppM][ievt%neppM]; // FIXME: assume process.nprocesses == 1 -#endif weightALL[iiter*nevt + ievt] = hstWeights[ievt]; } diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/mgOnGpuConfig.h b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/mgOnGpuConfig.h index 5475bbfe6c..23e45ca5b1 100644 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/mgOnGpuConfig.h +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/mgOnGpuConfig.h @@ -96,47 +96,79 @@ namespace mgOnGpu //const int ntpbMAX = 256; // AV Apr2021: why had I set this to 256? const int ntpbMAX = 1024; // NB: 512 is ok, but 1024 does fail with "too many resources requested for launch" - // Vector sizes for AOSOA memory layouts (GPU coalesced memory access, CPU SIMD vectorization) - // (these are all best kept as a compile-time constants: see issue #23) + // Alignment requirement for using reinterpret_cast with SIMD vectorized code + // (using reinterpret_cast with non aligned memory may lead to segmentation faults!) +#ifndef __CUDACC__ + constexpr int cppAlign = 64; // alignment requirement for SIMD vectorization (64-byte i.e. 512-bit) +#endif - // Number of Events Per Page in the momenta AOSOA memory layout + // C++ SIMD vectorization width (this will be used to set neppV) #ifdef __CUDACC__ #undef MGONGPU_CPPSIMD - // ----------------------------------------------------------------------------------------- - // --- GPUs: neppM must be a power of 2 times the number of fptype's in a 32-byte cacheline - // --- This is relevant to ensure coalesced access to momenta in global memory - // --- Note that neppR is hardcoded and may differ from neppM and neppV on some platforms - // ----------------------------------------------------------------------------------------- - //const int neppM = 64/sizeof(fptype); // 2x 32-byte GPU cache lines: 8 (DOUBLE) or 16 (FLOAT) - const int neppM = 32/sizeof(fptype); // (DEFAULT) 32-byte GPU cache line: 4 (DOUBLE) or 8 (FLOAT) - //const int neppM = 1; // *** NB: this is equivalent to AOS *** - //const int neppM = 32; // older default #else - // ----------------------------------------------------------------------------------------- - // --- CPUs: neppM must be exactly equal to the number of fptype's in a vector register - // --- [DEFAULT is 256-width AVX512 aka "512y" on CPUs, 32-byte as GPUs, faster than AVX2] - // --- The logic of the code requires the size neppV of fptype_v to be equal to neppM - // --- Note that neppR is hardcoded and may differ from neppM and neppV on some platforms - // ----------------------------------------------------------------------------------------- #if defined __AVX512VL__ -#define MGONGPU_CPPSIMD 1 #ifdef MGONGPU_PVW512 - const int neppM = 64/sizeof(fptype); // "512z" AVX512 with 512 width (512-bit ie 64-byte): 8 (DOUBLE) or 16 (FLOAT) + // "512z" AVX512 with 512 width (512-bit ie 64-byte): 8 (DOUBLE) or 16 (FLOAT) +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 8 #else - const int neppM = 32/sizeof(fptype); // "512y" AVX512 with 256 width (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [gcc DEFAULT] +#define MGONGPU_CPPSIMD 16 +#endif +#else + // "512y" AVX512 with 256 width (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [gcc DEFAULT] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 4 +#else +#define MGONGPU_CPPSIMD 8 +#endif #endif #elif defined __AVX2__ -#define MGONGPU_CPPSIMD 1 - const int neppM = 32/sizeof(fptype); // "avx2" AVX2 (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [clang DEFAULT] + // "avx2" AVX2 (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [clang DEFAULT] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 4 +#else +#define MGONGPU_CPPSIMD 8 +#endif #elif defined __SSE4_2__ -#define MGONGPU_CPPSIMD 1 - const int neppM = 16/sizeof(fptype); // "sse4" SSE4.2 (128-bit ie 16-byte): 2 (DOUBLE) or 4 (FLOAT) + // "sse4" SSE4.2 (128-bit ie 16-byte): 2 (DOUBLE) or 4 (FLOAT) +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 2 #else +#define MGONGPU_CPPSIMD 4 +#endif +#else + // "none" i.e. no SIMD (*** NB: this is equivalent to AOS ***) #undef MGONGPU_CPPSIMD - const int neppM = 1; // "none" i.e. no SIMD (*** NB: this is equivalent to AOS ***) #endif #endif + // Number of Events Per Page in the momenta AOSOA memory layout +#ifdef __CUDACC__ + // ----------------------------------------------------------------------------------------------- + // --- GPUs: neppM is best set to a power of 2 times the number of fptype's in a 32-byte cacheline + // --- This is relevant to ensure coalesced access to momenta in global memory + // --- Note that neppR is hardcoded and may differ from neppM and neppV on some platforms + // ----------------------------------------------------------------------------------------------- + //const int neppM = 64/sizeof(fptype); // 2x 32-byte GPU cache lines (512 bits): 8 (DOUBLE) or 16 (FLOAT) + const int neppM = 32/sizeof(fptype); // (DEFAULT) 32-byte GPU cache line (256 bits): 4 (DOUBLE) or 8 (FLOAT) + //const int neppM = 1; // *** NB: this is equivalent to AOS *** +#else + // ----------------------------------------------------------------------------------------------- + // --- CPUs: neppM is best set equal to the number of fptype's (neppV) in a vector register + // --- This is relevant to ensure faster access to momenta from C++ memory cache lines + // --- However, neppM is now decoupled from neppV (issue #176) and can be separately hardcoded + // --- In practice, neppR, neppM and neppV could now (in principle) all be different + // ----------------------------------------------------------------------------------------------- +#ifdef MGONGPU_CPPSIMD + const int neppM = MGONGPU_CPPSIMD; // (DEFAULT) neppM=neppV for optimal performance +#else + const int neppM = 1; // (DEFAULT) neppM=neppV for optimal performance (NB: this is equivalent to AOS) +#endif + //const int neppM = 64/sizeof(fptype); // maximum CPU vector width (512 bits): 8 (DOUBLE) or 16 (FLOAT) + //const int neppM = 32/sizeof(fptype); // lower CPU vector width (256 bits): 4 (DOUBLE) or 8 (FLOAT) + //const int neppM = 1; // *** NB: this is equivalent to AOS *** +#endif + // Number of Events Per Page in the random number AOSOA memory layout // *** NB Different values of neppR lead to different physics results: the *** // *** same 1d array is generated, but it is interpreted in different ways *** diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/mgOnGpuVectors.h b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/mgOnGpuVectors.h index e1da365663..4e975b2ca1 100644 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/mgOnGpuVectors.h +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/mgOnGpuVectors.h @@ -13,9 +13,13 @@ namespace mgOnGpu { + #ifdef MGONGPU_CPPSIMD - const int neppV = neppM; + const int neppV = MGONGPU_CPPSIMD; + + // SANITY CHECK: cppAlign must be a multiple of neppV * sizeof(fptype) + static_assert( mgOnGpu::cppAlign % ( neppV * sizeof(fptype) ) == 0 ); // --- Type definition (using vector compiler extensions: need -march=...) #ifdef __clang__ // https://clang.llvm.org/docs/LanguageExtensions.html#vectors-and-extended-vectors @@ -96,7 +100,7 @@ namespace mgOnGpu #else // MGONGPU_CPPSIMD not defined - const int neppV = 1; // Note: also neppM is equal to 1 + const int neppV = 1; #endif } @@ -114,7 +118,7 @@ using mgOnGpu::bool_v; inline std::ostream& operator<<( std::ostream& out, const bool_v& v ) { out << "{ " << v[0]; - for ( int i=1; i 0 ) && !( n & ( n - 1 ) ); }; + static_assert( ispoweroftwo( mgOnGpu::neppR ) ); + static_assert( ispoweroftwo( mgOnGpu::neppM ) ); + static_assert( ispoweroftwo( neppV ) ); #endif } @@ -173,9 +176,9 @@ namespace Proc #ifdef __CUDACC__ __global__ - void sigmaKin_getGoodHel( const fptype_sv* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM - fptype_sv* allMEs, // output: allMEs[npagM][neppM], final |M|^2 averaged over helicities - bool* isGoodHel ) // output: isGoodHel[ncomb] - device array + void sigmaKin_getGoodHel( const fptype* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM + fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities + bool* isGoodHel ) // output: isGoodHel[ncomb] - device array { const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid // FIXME: assume process.nprocesses == 1 for the moment (eventually: need a loop over processes here?) @@ -193,33 +196,35 @@ namespace Proc } } #else - void sigmaKin_getGoodHel( const fptype_sv* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM - fptype_sv* allMEs, // output: allMEs[npagM][neppM], final |M|^2 averaged over helicities - bool* isGoodHel // output: isGoodHel[ncomb] - device array - , const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + void sigmaKin_getGoodHel( const fptype* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM + fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities + bool* isGoodHel // output: isGoodHel[ncomb] - device array + , const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) { + assert( (size_t)(allmomenta) %% mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment + //assert( (size_t)(allMEs) %% mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment const int maxtry0 = ( neppV > 16 ? neppV : 16 ); // 16, but at least neppV (otherwise the npagV loop does not even start) - fptype_sv allMEsLast[maxtry0/neppV] = { 0 }; + fptype allMEsLast[maxtry0] = { 0 }; const int maxtry = std::min( maxtry0, nevt ); // 16, but at most nevt (avoid invalid memory access if nevt allMEs[nevt*nprocesses]?) __global__ - void sigmaKin( const fptype_sv* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM - fptype_sv* allMEs // output: allMEs[npagM][neppM], final |M|^2 averaged over helicities + void sigmaKin( const fptype* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM + fptype* allMEs // output: allMEs[nevt], |M|^2 final_avg_over_helicities #ifndef __CUDACC__ - , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) #endif ) { diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/process_h.inc b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/process_h.inc index f7df7ab214..b4e82a984e 100644 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/process_h.inc +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/process_h.inc @@ -47,11 +47,11 @@ namespace Proc //-------------------------------------------------------------------------- __global__ - void sigmaKin_getGoodHel( const fptype_sv* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM - fptype_sv* allMEs, // output: allMEs[npagM][neppM], final |M|^2 averaged over helicities - bool* isGoodHel // output: isGoodHel[ncomb] - device array + void sigmaKin_getGoodHel( const fptype* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM + fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities + bool* isGoodHel // output: isGoodHel[ncomb] - device array #ifndef __CUDACC__ - , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) #endif ); @@ -62,10 +62,10 @@ namespace Proc //-------------------------------------------------------------------------- __global__ - void sigmaKin( const fptype_sv* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM - fptype_sv* allMEs // output: allMEs[npagM][neppM], final |M|^2 averaged over helicities + void sigmaKin( const fptype* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM + fptype* allMEs // output: allMEs[nevt], |M|^2 final_avg_over_helicities #ifndef __CUDACC__ - , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) #endif ); diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/process_matrix.inc b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/process_matrix.inc index f2429fb7d4..8c32179ca6 100644 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/process_matrix.inc +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/process_matrix.inc @@ -28,13 +28,27 @@ #ifdef __CUDACC__ const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid allMEs[ievt] += deltaMEs; - //printf( "calculate_wavefunction: %%6d %%2d %%f\n", ievt, ihel, allMEs[ievt] ); + //if ( cNGoodHel > 0 ) printf( "calculate_wavefunction: %%6d %%2d %%f\n", ievt, ihel, allMEs[ievt] ); +#else +#ifdef MGONGPU_CPPSIMD + if ( isAligned_allMEs ) + { + *reinterpret_cast( &( allMEs[ipagV*neppV] ) ) += deltaMEs; + } + else + { + for ( int ieppV=0; ieppV 0 ) + // for ( int ieppV=0; ieppV 0 ) printf( "calculate_wavefunction: %%6d %%2d %%f\n", ipagV, ihel, allMEs[ipagV] ); +#endif #endif } - mgDebug( 1, __FUNCTION__ ); return; } diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/process_sigmaKin_function.inc b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/process_sigmaKin_function.inc index 24816293b5..73b2f16775 100644 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/process_sigmaKin_function.inc +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/process_sigmaKin_function.inc @@ -7,7 +7,8 @@ const int npagV = nevt/neppV; for ( int ipagV = 0; ipagV < npagV; ++ipagV ) { - allMEs[ipagV] = fptype_sv{ 0 }; + for ( int ieppV=0; ieppV #include @@ -22,7 +22,7 @@ namespace rambo2toNm0 // [NB: the output buffer includes both initial and final momenta, but only initial momenta are filled in] __global__ void getMomentaInitial( const fptype energy, // input: energy - fptype_sv momenta1d[] // output: momenta as AOSOA[npagM][npar][4][neppM] + fptype momenta1d[] // output: momenta as AOSOA[npagM][npar][4][neppM] #ifndef __CUDACC__ , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) #endif @@ -63,7 +63,7 @@ namespace rambo2toNm0 __global__ void getMomentaFinal( const fptype energy, // input: energy const fptype rnarray1d[], // input: random numbers in [0,1] as AOSOA[npagR][nparf][4][neppR] - fptype_sv momenta1d[], // output: momenta as AOSOA[npagM][npar][4][neppM] + fptype momenta1d[], // output: momenta as AOSOA[npagM][npar][4][neppM] fptype wgts[] // output: weights[nevt] #ifndef __CUDACC__ , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/rambo.h b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/rambo.h index 030526759f..9d9e876fe3 100644 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/rambo.h +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/rambo.h @@ -3,7 +3,7 @@ #include "mgOnGpuConfig.h" #include "mgOnGpuTypes.h" -#include "mgOnGpuVectors.h" +//#include "mgOnGpuVectors.h" #include @@ -47,7 +47,7 @@ namespace rambo2toNm0 // [NB: the output buffer includes both initial and final momenta, but only initial momenta are filled in] __global__ void getMomentaInitial( const fptype energy, // input: energy - fptype_sv momenta1d[] // output: momenta as AOSOA[npagM][npar][4][neppM] + fptype momenta1d[] // output: momenta as AOSOA[npagM][npar][4][neppM] #ifndef __CUDACC__ , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) #endif @@ -60,7 +60,7 @@ namespace rambo2toNm0 __global__ void getMomentaFinal( const fptype energy, // input: energy const fptype rnarray1d[], // input: random numbers in [0,1] as AOSOA[npagR][nparf][4][neppR] - fptype_sv momenta1d[], // output: momenta as AOSOA[npagM][npar][4][neppM] + fptype momenta1d[], // output: momenta as AOSOA[npagM][npar][4][neppM] fptype wgts[] // output: weights[nevt] #ifndef __CUDACC__ , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/runTest.cc b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/runTest.cc index d92ffe4e7a..a4f3519434 100644 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/runTest.cc +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/runTest.cc @@ -21,7 +21,7 @@ template using unique_ptr_host = std::unique_ptr>; #else template -using unique_ptr_host = std::unique_ptr; +using unique_ptr_host = std::unique_ptr>; #endif struct CUDA_CPU_TestBase : public TestDriverBase { @@ -47,11 +47,11 @@ struct CPUTest : public CUDA_CPU_TestBase { // Struct data members (process, and memory structures for random numbers, momenta, matrix elements and weights on host and device) // [NB the hst/dev memory arrays must be initialised in the constructor, see issue #290] Proc::CPPProcess process; - unique_ptr_host hstRnarray; - unique_ptr_host hstMomenta; - unique_ptr_host hstIsGoodHel; - unique_ptr_host hstWeights; - unique_ptr_host hstMEs; + unique_ptr_host hstRnarray; + unique_ptr_host hstMomenta; + unique_ptr_host hstIsGoodHel; + unique_ptr_host hstWeights; + unique_ptr_host hstMEs; // Create a process object // Read param_card and set parameters @@ -61,11 +61,11 @@ struct CPUTest : public CUDA_CPU_TestBase { CPUTest( const std::string& refFileName ) : CUDA_CPU_TestBase( refFileName ), process(niter, gpublocks, gputhreads, /*verbose=*/false), - hstRnarray { hstMakeUnique( nRnarray ) }, // AOSOA[npagR][nparf][np4][neppR] - hstMomenta { hstMakeUnique( nMomenta ) }, // AOSOA[npagM][npar][np4][neppM] - hstIsGoodHel{ hstMakeUnique( mgOnGpu::ncomb ) }, - hstWeights { hstMakeUnique( nWeights ) }, - hstMEs { hstMakeUnique( nMEs ) } // AOSOA[npagM][neppM] + hstRnarray { hstMakeUnique( nRnarray ) }, // AOSOA[npagR][nparf][np4][neppR] + hstMomenta { hstMakeUnique( nMomenta ) }, // AOSOA[npagM][npar][np4][neppM] + hstIsGoodHel{ hstMakeUnique( mgOnGpu::ncomb ) }, + hstWeights { hstMakeUnique( nWeights ) }, + hstMEs { hstMakeUnique( nMEs ) } // ARRAY[nevt] { process.initProc("../../Cards/param_card.dat"); } @@ -106,19 +106,11 @@ struct CPUTest : public CUDA_CPU_TestBase { assert(particle < npar); const auto ipagM = evtNo / neppM; // #eventpage in this iteration const auto ieppM = evtNo % neppM; // #event in the current eventpage in this iteration -#ifndef MGONGPU_CPPSIMD return hstMomenta[ipagM*npar*np4*neppM + particle*np4*neppM + component*neppM + ieppM]; -#else - return hstMomenta[ipagM*npar*np4 + particle*np4 + component][ieppM]; -#endif }; fptype getMatrixElement(std::size_t ievt) const override { -#ifndef MGONGPU_CPPSIMD return hstMEs[ievt]; -#else - return hstMEs[ievt/neppV][ievt%neppV]; -#endif } }; @@ -161,12 +153,12 @@ struct CUDATest : public CUDA_CPU_TestBase { hstMomenta { hstMakeUnique( nMomenta ) }, // AOSOA[npagM][npar][np4][neppM] (nevt=npagM*neppM) hstIsGoodHel{ hstMakeUnique( mgOnGpu::ncomb ) }, hstWeights { hstMakeUnique( nWeights ) }, - hstMEs { hstMakeUnique( nMEs ) }, + hstMEs { hstMakeUnique( nMEs ) }, // ARRAY[nevt] devRnarray { devMakeUnique( nRnarray ) }, // AOSOA[npagR][nparf][np4][neppR] (nevt=npagR*neppR) - devMomenta { devMakeUnique( nMomenta ) }, + devMomenta { devMakeUnique( nMomenta ) }, // AOSOA[npagM][npar][np4][neppM] (nevt=npagM*neppM) devIsGoodHel{ devMakeUnique( mgOnGpu::ncomb ) }, devWeights { devMakeUnique( nWeights ) }, - devMEs { devMakeUnique( nMEs ) } + devMEs { devMakeUnique( nMEs ) } // ARRAY[nevt] { process.initProc("../../Cards/param_card.dat"); } diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/testxxx.cc b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/testxxx.cc index 6df896c75f..9ae06a7404 100644 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/testxxx.cc +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/testxxx.cc @@ -31,8 +31,8 @@ TEST( XTESTID_CPU( MG_EPOCH_PROCESS_ID ), testxxx ) assert( nevt % neppM == 0 ); // nevt must be a multiple of neppM // Fill in the input momenta const int nMomenta = np4 * npar * nevt; - auto hstMomenta = hstMakeUnique( nMomenta ); // AOSOA[npagM][npar=4][np4=4][neppM] - const fptype par0[np4 * nevt] = // AOS[nevt][np4] + auto hstMomenta = hstMakeUnique( nMomenta ); // AOSOA[npagM][npar=4][np4=4][neppM] + const fptype par0[np4 * nevt] = // AOS[nevt][np4] { 500, 0, 0, 500, // #0 (m=0 pT=0 E=pz>0) 500, 0, 0, -500, // #1 (m=0 pT=0 -E=pz<0) 500, 300, 400, 0, // #2 (m=0 pT>0 pz=0) @@ -72,11 +72,7 @@ TEST( XTESTID_CPU( MG_EPOCH_PROCESS_ID ), testxxx ) { const int ipagM = ievt/neppM; // #eventpage in this iteration const int ieppM = ievt%neppM; // #event in the current eventpage in this iteration -#ifdef MGONGPU_CPPSIMD - hstMomenta[ipagM*npar*np4 + ipar*np4 + ip4][ieppM] = par0[ievt*np4 + ip4]; // AOS to AOSOA -#else hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + ip4*neppM + ieppM] = par0[ievt*np4 + ip4]; // AOS to AOSOA -#endif } } // Expected output wavefunctions diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/model_handling.py b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/model_handling.py index e3266f57da..79bc5ead17 100644 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/model_handling.py +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/model_handling.py @@ -810,10 +810,10 @@ def get_all_sigmaKin_lines(self, color_amplitudes, class_name): __device__ INLINE void calculate_wavefunctions( int ihel, - const fptype_sv* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM - fptype_sv* allMEs // output: allMEs[npagM][neppM], final |M|^2 averaged over helicities + const fptype* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM + fptype* allMEs // output: allMEs[nevt], |M|^2 running_sum_over_helicities #ifndef __CUDACC__ - , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) #endif ) //ALWAYS_INLINE // attributes are not permitted in a function definition @@ -837,6 +837,9 @@ def get_all_sigmaKin_lines(self, color_amplitudes, class_name): // === Calculate wavefunctions and amplitudes for all diagrams in all processes - Loop over nevt events === #ifndef __CUDACC__ const int npagV = nevt / neppV; +#ifdef MGONGPU_CPPSIMD + const bool isAligned_allMEs = ( (size_t)(allMEs) % mgOnGpu::cppAlign == 0 ); // require SIMD-friendly alignment by at least neppV*sizeof(fptype) +#endif // ** START LOOP ON IPAGV ** #ifdef _OPENMP // (NB gcc9 or higher, or clang, is required) @@ -844,7 +847,11 @@ def get_all_sigmaKin_lines(self, color_amplitudes, class_name): // - shared: as the name says // - private: give each thread its own copy, without initialising // - firstprivate: give each thread its own copy, and initialise with value from outside +#ifdef MGONGPU_CPPSIMD +#pragma omp parallel for default(none) shared(allmomenta,allMEs,cHel,cIPC,cIPD,ihel,npagV,isAligned_allMEs) private (amp_sv,w_sv,jamp_sv) +#else #pragma omp parallel for default(none) shared(allmomenta,allMEs,cHel,cIPC,cIPD,ihel,npagV) private (amp_sv,w_sv,jamp_sv) +#endif #endif for ( int ipagV = 0; ipagV < npagV; ++ipagV ) #endif""") From e2dfcb1f5b587fa8b0c7425e5024cbf3117fff87 Mon Sep 17 00:00:00 2001 From: Andrea Valassi Date: Wed, 1 Dec 2021 10:01:51 +0100 Subject: [PATCH 11/34] [klas5X] regenerate eemumu.auto - now identical to eemumu manual --- .../CODEGEN_cudacpp_ee_mumu_log.txt | 20 +- .../ee_mumu.auto/SubProcesses/Memory.h | 19 +- .../P1_Sigma_sm_epem_mupmum/CPPProcess.cc | 82 +++++--- .../P1_Sigma_sm_epem_mupmum/CPPProcess.h | 14 +- .../P1_Sigma_sm_epem_mupmum/check_sa.cc | 35 +--- .../ee_mumu.auto/SubProcesses/runTest.cc | 36 ++-- .../ee_mumu.auto/SubProcesses/testxxx.cc | 8 +- epochX/cudacpp/ee_mumu.auto/src/HelAmps_sm.cc | 183 ++++++++++++++---- epochX/cudacpp/ee_mumu.auto/src/HelAmps_sm.h | 20 +- .../cudacpp/ee_mumu.auto/src/mgOnGpuConfig.h | 84 +++++--- .../cudacpp/ee_mumu.auto/src/mgOnGpuVectors.h | 16 +- epochX/cudacpp/ee_mumu.auto/src/rambo.cc | 6 +- epochX/cudacpp/ee_mumu.auto/src/rambo.h | 6 +- 13 files changed, 345 insertions(+), 184 deletions(-) diff --git a/epochX/cudacpp/ee_mumu.auto/CODEGEN_cudacpp_ee_mumu_log.txt b/epochX/cudacpp/ee_mumu.auto/CODEGEN_cudacpp_ee_mumu_log.txt index dbc19f9dba..9c07f0f53d 100644 --- a/epochX/cudacpp/ee_mumu.auto/CODEGEN_cudacpp_ee_mumu_log.txt +++ b/epochX/cudacpp/ee_mumu.auto/CODEGEN_cudacpp_ee_mumu_log.txt @@ -50,7 +50,7 @@ generate e+ e- > mu+ mu- No model currently active, so we import the Standard Model INFO: load particles INFO: load vertices -DEBUG: model prefixing takes 0.006849765777587891  +DEBUG: model prefixing takes 0.0068590641021728516  INFO: Restrict model sm with file models/sm/restrict_default.dat . INFO: Change particles name to pass to MG5 convention DEBUG: True [misc.py at line 2192]  @@ -81,16 +81,16 @@ INFO: Processing color information for process: e+ e- > mu+ mu- @1 DEBUG: type(subproc_group)= [output.py at line 139]  DEBUG: type(fortran_model)= [output.py at line 140]  INFO: Creating files in directory /data/avalassi/GPU2020/MG5aMC/2.7.0_gpu/CODEGEN_cudacpp_ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum -DEBUG: Entering PLUGIN_OneProcessExporter.generate_process_files [model_handling.py at line 886]  +DEBUG: Entering PLUGIN_OneProcessExporter.generate_process_files [model_handling.py at line 893]  FileWriter for /data/avalassi/GPU2020/MG5aMC/2.7.0_gpu/CODEGEN_cudacpp_ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/./CPPProcess.h -DEBUG: Entering PLUGIN_OneProcessExporter.write_process_h_file [model_handling.py at line 935]  +DEBUG: Entering PLUGIN_OneProcessExporter.write_process_h_file [model_handling.py at line 942]  FileWriter for /data/avalassi/GPU2020/MG5aMC/2.7.0_gpu/CODEGEN_cudacpp_ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/./CPPProcess.cc -DEBUG: Entering PLUGIN_OneProcessExporter.write_process_cc_file [model_handling.py at line 957]  +DEBUG: Entering PLUGIN_OneProcessExporter.write_process_cc_file [model_handling.py at line 964]  DEBUG: only one Matrix-element supported?  INFO: Created files CPPProcess.h and CPPProcess.cc in directory /data/avalassi/GPU2020/MG5aMC/2.7.0_gpu/CODEGEN_cudacpp_ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/. -DEBUG: Entering PLUGIN_OneProcessExporter.edit_check_sa [model_handling.py at line 900]  -DEBUG: Entering PLUGIN_OneProcessExporter.edit_mgonGPU [model_handling.py at line 915]  -DEBUG: Entering PLUGIN_OneProcessExporter.edit_processidfile [model_handling.py at line 922]  +DEBUG: Entering PLUGIN_OneProcessExporter.edit_check_sa [model_handling.py at line 907]  +DEBUG: Entering PLUGIN_OneProcessExporter.edit_mgonGPU [model_handling.py at line 922]  +DEBUG: Entering PLUGIN_OneProcessExporter.edit_processidfile [model_handling.py at line 929]  Generated helas calls for 1 subprocesses (2 diagrams) in 0.005 s DEBUG: Entering PLUGIN_ProcessExporter.convert_model (create the model) [output.py at line 145]  ALOHA: aloha creates routines (starting by FFV1) @@ -121,6 +121,6 @@ INFO: /data/avalassi/GPU2020/MG5aMC/2.7.0_gpu/CODEGEN_cudacpp_ee_mumu/src/. and DEBUG: Entering PLUGIN_ProcessExporter.finalize [output.py at line 154]  quit -real 0m5.179s -user 0m0.933s -sys 0m0.139s +real 0m18.430s +user 0m0.923s +sys 0m0.138s diff --git a/epochX/cudacpp/ee_mumu.auto/SubProcesses/Memory.h b/epochX/cudacpp/ee_mumu.auto/SubProcesses/Memory.h index d7d1c7d4da..a9c08d3dd0 100644 --- a/epochX/cudacpp/ee_mumu.auto/SubProcesses/Memory.h +++ b/epochX/cudacpp/ee_mumu.auto/SubProcesses/Memory.h @@ -1,8 +1,9 @@ /* - * memory.h + * Memory.h * * Created on: 19.11.2020 * Author: shageboeck + * Modified: avalassi (vectors, alignment) */ #ifndef MEMORY_H @@ -46,13 +47,25 @@ std::unique_ptr> hstMakeUnique(std::size_t N) { #else +template +struct CppHstDeleter { + void operator()(T* mem) { + ::operator delete( mem, std::align_val_t{ mgOnGpu::cppAlign } ); + } +}; + template inline -std::unique_ptr hstMakeUnique(std::size_t N) { return std::unique_ptr{ new T[N]() }; }; +std::unique_ptr> hstMakeUnique(std::size_t N) { + // See https://www.bfilipek.com/2019/08/newnew-align.html#requesting-different-alignment + return std::unique_ptr>{ new( std::align_val_t{ mgOnGpu::cppAlign } ) T[N]() }; +}; #ifdef MGONGPU_CPPSIMD template<> inline -std::unique_ptr hstMakeUnique(std::size_t N) { return std::unique_ptr{ new fptype_v[N/neppV]() }; }; +std::unique_ptr> hstMakeUnique(std::size_t N) { + return std::unique_ptr>{ new( std::align_val_t{ mgOnGpu::cppAlign } ) fptype_v[N/neppV]() }; +}; #endif diff --git a/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc b/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc index 906c8523e2..d49bf181ce 100644 --- a/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc +++ b/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc @@ -71,10 +71,10 @@ namespace Proc __device__ INLINE void calculate_wavefunctions( int ihel, - const fptype_sv* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM - fptype_sv* allMEs // output: allMEs[npagM][neppM], final |M|^2 averaged over helicities + const fptype* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM + fptype* allMEs // output: allMEs[nevt], |M|^2 running_sum_over_helicities #ifndef __CUDACC__ - , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) #endif ) //ALWAYS_INLINE // attributes are not permitted in a function definition @@ -99,6 +99,9 @@ namespace Proc // === Calculate wavefunctions and amplitudes for all diagrams in all processes - Loop over nevt events === #ifndef __CUDACC__ const int npagV = nevt / neppV; +#ifdef MGONGPU_CPPSIMD + const bool isAligned_allMEs = ( (size_t)(allMEs) % mgOnGpu::cppAlign == 0 ); // require SIMD-friendly alignment by at least neppV*sizeof(fptype) +#endif // ** START LOOP ON IPAGV ** #ifdef _OPENMP // (NB gcc9 or higher, or clang, is required) @@ -106,7 +109,11 @@ namespace Proc // - shared: as the name says // - private: give each thread its own copy, without initialising // - firstprivate: give each thread its own copy, and initialise with value from outside +#ifdef MGONGPU_CPPSIMD +#pragma omp parallel for default(none) shared(allmomenta,allMEs,cHel,cIPC,cIPD,ihel,npagV,isAligned_allMEs) private (amp_sv,w_sv,jamp_sv) +#else #pragma omp parallel for default(none) shared(allmomenta,allMEs,cHel,cIPC,cIPD,ihel,npagV) private (amp_sv,w_sv,jamp_sv) +#endif #endif for ( int ipagV = 0; ipagV < npagV; ++ipagV ) #endif @@ -193,13 +200,27 @@ namespace Proc #ifdef __CUDACC__ const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid allMEs[ievt] += deltaMEs; - //printf( "calculate_wavefunction: %6d %2d %f\n", ievt, ihel, allMEs[ievt] ); + //if ( cNGoodHel > 0 ) printf( "calculate_wavefunction: %6d %2d %f\n", ievt, ihel, allMEs[ievt] ); +#else +#ifdef MGONGPU_CPPSIMD + if ( isAligned_allMEs ) + { + *reinterpret_cast( &( allMEs[ipagV*neppV] ) ) += deltaMEs; + } + else + { + for ( int ieppV=0; ieppV 0 ) + // for ( int ieppV=0; ieppV 0 ) printf( "calculate_wavefunction: %6d %2d %f\n", ipagV, ihel, allMEs[ipagV] ); +#endif #endif } - mgDebug( 1, __FUNCTION__ ); return; } @@ -243,10 +264,13 @@ namespace Proc memcpy( cHel, tHel, ncomb * mgOnGpu::npar * sizeof(short) ); #endif // SANITY CHECK: GPU memory usage may be based on casts of fptype[2] to cxtype - assert( sizeof(cxtype) == 2 * sizeof(fptype) ); + static_assert( sizeof(cxtype) == 2 * sizeof(fptype) ); #ifndef __CUDACC__ - // SANITY CHECK: momenta AOSOA uses vectors with the same size as fptype_v - assert( neppV == mgOnGpu::neppM ); + // SANITY CHECK: check that neppR, neppM and neppV are powers of two (https://stackoverflow.com/a/108360) + auto ispoweroftwo = []( int n ) { return ( n > 0 ) && !( n & ( n - 1 ) ); }; + static_assert( ispoweroftwo( mgOnGpu::neppR ) ); + static_assert( ispoweroftwo( mgOnGpu::neppM ) ); + static_assert( ispoweroftwo( neppV ) ); #endif } @@ -356,9 +380,9 @@ namespace Proc #ifdef __CUDACC__ __global__ - void sigmaKin_getGoodHel( const fptype_sv* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM - fptype_sv* allMEs, // output: allMEs[npagM][neppM], final |M|^2 averaged over helicities - bool* isGoodHel ) // output: isGoodHel[ncomb] - device array + void sigmaKin_getGoodHel( const fptype* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM + fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities + bool* isGoodHel ) // output: isGoodHel[ncomb] - device array { const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid // FIXME: assume process.nprocesses == 1 for the moment (eventually: need a loop over processes here?) @@ -376,33 +400,35 @@ namespace Proc } } #else - void sigmaKin_getGoodHel( const fptype_sv* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM - fptype_sv* allMEs, // output: allMEs[npagM][neppM], final |M|^2 averaged over helicities - bool* isGoodHel // output: isGoodHel[ncomb] - device array - , const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + void sigmaKin_getGoodHel( const fptype* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM + fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities + bool* isGoodHel // output: isGoodHel[ncomb] - device array + , const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) { + assert( (size_t)(allmomenta) % mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment + //assert( (size_t)(allMEs) % mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment const int maxtry0 = ( neppV > 16 ? neppV : 16 ); // 16, but at least neppV (otherwise the npagV loop does not even start) - fptype_sv allMEsLast[maxtry0/neppV] = { 0 }; + fptype allMEsLast[maxtry0] = { 0 }; const int maxtry = std::min( maxtry0, nevt ); // 16, but at most nevt (avoid invalid memory access if nevt allMEs[nevt*nprocesses]?) __global__ - void sigmaKin( const fptype_sv* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM - fptype_sv* allMEs // output: allMEs[npagM][neppM], final |M|^2 averaged over helicities + void sigmaKin( const fptype* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM + fptype* allMEs // output: allMEs[nevt], |M|^2 final_avg_over_helicities #ifndef __CUDACC__ - , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) #endif ) { @@ -472,7 +498,8 @@ namespace Proc const int npagV = nevt/neppV; for ( int ipagV = 0; ipagV < npagV; ++ipagV ) { - allMEs[ipagV] = fptype_sv{ 0 }; + for ( int ieppV=0; ieppV( nRnarray ); // AOSOA[npagR][nparf][np4][neppR] (NB: nevt=npagR*neppR) + auto hstRnarray = hstMakeUnique( nRnarray ); // AOSOA[npagR][nparf][np4][neppR] (NB: nevt=npagR*neppR) #endif - auto hstMomenta = hstMakeUnique( nMomenta ); // AOSOA[npagM][npar][np4][neppM] (NB: nevt=npagM*neppM) - auto hstIsGoodHel = hstMakeUnique( ncomb ); - auto hstWeights = hstMakeUnique( nWeights ); - auto hstMEs = hstMakeUnique( nMEs ); // AOSOA[npagM][neppM] (NB: nevt=npagM*neppM) + auto hstMomenta = hstMakeUnique( nMomenta ); // AOSOA[npagM][npar][np4][neppM] (NB: nevt=npagM*neppM) + auto hstIsGoodHel = hstMakeUnique( ncomb ); + auto hstWeights = hstMakeUnique( nWeights ); + auto hstMEs = hstMakeUnique( nMEs ); // ARRAY[nevt] #ifdef __CUDACC__ - auto devRnarray = devMakeUnique( nRnarray ); // AOSOA[npagR][nparf][np4][neppR] (NB: nevt=npagR*neppR) - auto devMomenta = devMakeUnique( nMomenta ); // AOSOA[npagM][npar][np4][neppM] (NB: nevt=npagM*neppM) - auto devIsGoodHel = devMakeUnique( ncomb ); - auto devWeights = devMakeUnique( nWeights ); - auto devMEs = devMakeUnique( nMEs ); // AOSOA[npagM][neppM] (NB: nevt=npagM*neppM) + auto devRnarray = devMakeUnique( nRnarray ); // AOSOA[npagR][nparf][np4][neppR] (NB: nevt=npagR*neppR) + auto devMomenta = devMakeUnique( nMomenta ); // AOSOA[npagM][npar][np4][neppM] (NB: nevt=npagM*neppM) + auto devIsGoodHel = devMakeUnique( ncomb ); + auto devWeights = devMakeUnique( nWeights ); + auto devMEs = devMakeUnique( nMEs ); // ARRAY[nevt] #if defined MGONGPU_CURAND_ONHOST or defined MGONGPU_COMMONRAND_ONHOST const int nbytesRnarray = nRnarray * sizeof(fptype); @@ -553,37 +553,22 @@ int main(int argc, char **argv) // NB: 'setw' affects only the next field (of any type) std::cout << std::scientific // fixed format: affects all floats (default precision: 6) << std::setw(4) << ipar + 1 -#ifndef MGONGPU_CPPSIMD << std::setw(14) << hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + 0*neppM + ieppM] // AOSOA[ipagM][ipar][0][ieppM] << std::setw(14) << hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + 1*neppM + ieppM] // AOSOA[ipagM][ipar][1][ieppM] << std::setw(14) << hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + 2*neppM + ieppM] // AOSOA[ipagM][ipar][2][ieppM] << std::setw(14) << hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + 3*neppM + ieppM] // AOSOA[ipagM][ipar][3][ieppM] -#else - << std::setw(14) << hstMomenta[ipagM*npar*np4 + ipar*np4 + 0][ieppM] // AOSOA[ipagM][ipar][0][ieppM] - << std::setw(14) << hstMomenta[ipagM*npar*np4 + ipar*np4 + 1][ieppM] // AOSOA[ipagM][ipar][1][ieppM] - << std::setw(14) << hstMomenta[ipagM*npar*np4 + ipar*np4 + 2][ieppM] // AOSOA[ipagM][ipar][2][ieppM] - << std::setw(14) << hstMomenta[ipagM*npar*np4 + ipar*np4 + 3][ieppM] // AOSOA[ipagM][ipar][3][ieppM] -#endif << std::endl << std::defaultfloat; // default format: affects all floats } std::cout << std::string(SEP79, '-') << std::endl; // Display matrix elements std::cout << " Matrix element = " -#ifndef MGONGPU_CPPSIMD << hstMEs[ievt] -#else - << hstMEs[ievt/neppM][ievt%neppM] -#endif << " GeV^" << meGeVexponent << std::endl; // FIXME: assume process.nprocesses == 1 std::cout << std::string(SEP79, '-') << std::endl; } // Fill the arrays with ALL MEs and weights -#ifndef MGONGPU_CPPSIMD matrixelementALL[iiter*nevt + ievt] = hstMEs[ievt]; // FIXME: assume process.nprocesses == 1 -#else - matrixelementALL[iiter*nevt + ievt] = hstMEs[ievt/neppM][ievt%neppM]; // FIXME: assume process.nprocesses == 1 -#endif weightALL[iiter*nevt + ievt] = hstWeights[ievt]; } diff --git a/epochX/cudacpp/ee_mumu.auto/SubProcesses/runTest.cc b/epochX/cudacpp/ee_mumu.auto/SubProcesses/runTest.cc index d92ffe4e7a..a4f3519434 100644 --- a/epochX/cudacpp/ee_mumu.auto/SubProcesses/runTest.cc +++ b/epochX/cudacpp/ee_mumu.auto/SubProcesses/runTest.cc @@ -21,7 +21,7 @@ template using unique_ptr_host = std::unique_ptr>; #else template -using unique_ptr_host = std::unique_ptr; +using unique_ptr_host = std::unique_ptr>; #endif struct CUDA_CPU_TestBase : public TestDriverBase { @@ -47,11 +47,11 @@ struct CPUTest : public CUDA_CPU_TestBase { // Struct data members (process, and memory structures for random numbers, momenta, matrix elements and weights on host and device) // [NB the hst/dev memory arrays must be initialised in the constructor, see issue #290] Proc::CPPProcess process; - unique_ptr_host hstRnarray; - unique_ptr_host hstMomenta; - unique_ptr_host hstIsGoodHel; - unique_ptr_host hstWeights; - unique_ptr_host hstMEs; + unique_ptr_host hstRnarray; + unique_ptr_host hstMomenta; + unique_ptr_host hstIsGoodHel; + unique_ptr_host hstWeights; + unique_ptr_host hstMEs; // Create a process object // Read param_card and set parameters @@ -61,11 +61,11 @@ struct CPUTest : public CUDA_CPU_TestBase { CPUTest( const std::string& refFileName ) : CUDA_CPU_TestBase( refFileName ), process(niter, gpublocks, gputhreads, /*verbose=*/false), - hstRnarray { hstMakeUnique( nRnarray ) }, // AOSOA[npagR][nparf][np4][neppR] - hstMomenta { hstMakeUnique( nMomenta ) }, // AOSOA[npagM][npar][np4][neppM] - hstIsGoodHel{ hstMakeUnique( mgOnGpu::ncomb ) }, - hstWeights { hstMakeUnique( nWeights ) }, - hstMEs { hstMakeUnique( nMEs ) } // AOSOA[npagM][neppM] + hstRnarray { hstMakeUnique( nRnarray ) }, // AOSOA[npagR][nparf][np4][neppR] + hstMomenta { hstMakeUnique( nMomenta ) }, // AOSOA[npagM][npar][np4][neppM] + hstIsGoodHel{ hstMakeUnique( mgOnGpu::ncomb ) }, + hstWeights { hstMakeUnique( nWeights ) }, + hstMEs { hstMakeUnique( nMEs ) } // ARRAY[nevt] { process.initProc("../../Cards/param_card.dat"); } @@ -106,19 +106,11 @@ struct CPUTest : public CUDA_CPU_TestBase { assert(particle < npar); const auto ipagM = evtNo / neppM; // #eventpage in this iteration const auto ieppM = evtNo % neppM; // #event in the current eventpage in this iteration -#ifndef MGONGPU_CPPSIMD return hstMomenta[ipagM*npar*np4*neppM + particle*np4*neppM + component*neppM + ieppM]; -#else - return hstMomenta[ipagM*npar*np4 + particle*np4 + component][ieppM]; -#endif }; fptype getMatrixElement(std::size_t ievt) const override { -#ifndef MGONGPU_CPPSIMD return hstMEs[ievt]; -#else - return hstMEs[ievt/neppV][ievt%neppV]; -#endif } }; @@ -161,12 +153,12 @@ struct CUDATest : public CUDA_CPU_TestBase { hstMomenta { hstMakeUnique( nMomenta ) }, // AOSOA[npagM][npar][np4][neppM] (nevt=npagM*neppM) hstIsGoodHel{ hstMakeUnique( mgOnGpu::ncomb ) }, hstWeights { hstMakeUnique( nWeights ) }, - hstMEs { hstMakeUnique( nMEs ) }, + hstMEs { hstMakeUnique( nMEs ) }, // ARRAY[nevt] devRnarray { devMakeUnique( nRnarray ) }, // AOSOA[npagR][nparf][np4][neppR] (nevt=npagR*neppR) - devMomenta { devMakeUnique( nMomenta ) }, + devMomenta { devMakeUnique( nMomenta ) }, // AOSOA[npagM][npar][np4][neppM] (nevt=npagM*neppM) devIsGoodHel{ devMakeUnique( mgOnGpu::ncomb ) }, devWeights { devMakeUnique( nWeights ) }, - devMEs { devMakeUnique( nMEs ) } + devMEs { devMakeUnique( nMEs ) } // ARRAY[nevt] { process.initProc("../../Cards/param_card.dat"); } diff --git a/epochX/cudacpp/ee_mumu.auto/SubProcesses/testxxx.cc b/epochX/cudacpp/ee_mumu.auto/SubProcesses/testxxx.cc index 6df896c75f..9ae06a7404 100644 --- a/epochX/cudacpp/ee_mumu.auto/SubProcesses/testxxx.cc +++ b/epochX/cudacpp/ee_mumu.auto/SubProcesses/testxxx.cc @@ -31,8 +31,8 @@ TEST( XTESTID_CPU( MG_EPOCH_PROCESS_ID ), testxxx ) assert( nevt % neppM == 0 ); // nevt must be a multiple of neppM // Fill in the input momenta const int nMomenta = np4 * npar * nevt; - auto hstMomenta = hstMakeUnique( nMomenta ); // AOSOA[npagM][npar=4][np4=4][neppM] - const fptype par0[np4 * nevt] = // AOS[nevt][np4] + auto hstMomenta = hstMakeUnique( nMomenta ); // AOSOA[npagM][npar=4][np4=4][neppM] + const fptype par0[np4 * nevt] = // AOS[nevt][np4] { 500, 0, 0, 500, // #0 (m=0 pT=0 E=pz>0) 500, 0, 0, -500, // #1 (m=0 pT=0 -E=pz<0) 500, 300, 400, 0, // #2 (m=0 pT>0 pz=0) @@ -72,11 +72,7 @@ TEST( XTESTID_CPU( MG_EPOCH_PROCESS_ID ), testxxx ) { const int ipagM = ievt/neppM; // #eventpage in this iteration const int ieppM = ievt%neppM; // #event in the current eventpage in this iteration -#ifdef MGONGPU_CPPSIMD - hstMomenta[ipagM*npar*np4 + ipar*np4 + ip4][ieppM] = par0[ievt*np4 + ip4]; // AOS to AOSOA -#else hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + ip4*neppM + ieppM] = par0[ievt*np4 + ip4]; // AOS to AOSOA -#endif } } // Expected output wavefunctions diff --git a/epochX/cudacpp/ee_mumu.auto/src/HelAmps_sm.cc b/epochX/cudacpp/ee_mumu.auto/src/HelAmps_sm.cc index a1cdb5ab30..3ec3d545c3 100644 --- a/epochX/cudacpp/ee_mumu.auto/src/HelAmps_sm.cc +++ b/epochX/cudacpp/ee_mumu.auto/src/HelAmps_sm.cc @@ -23,8 +23,8 @@ namespace MG5_sm //-------------------------------------------------------------------------- -#ifdef __CUDACC__ - // Return by reference + // Decode momentum AOSOA: compute address of fptype for the given particle, 4-momentum component and event + // Return the fptype by reference (equivalent to returning its memory address) __device__ inline const fptype& pIparIp4Ievt( const fptype* momenta, // input: momenta as AOSOA[npagM][npar][4][neppM] const int ipar, @@ -33,38 +33,149 @@ namespace MG5_sm { using mgOnGpu::np4; using mgOnGpu::npar; - const int neppM = mgOnGpu::neppM; // AOSOA layout: constant at compile-time - const int ipagM = ievt/neppM; // #eventpage in this iteration - const int ieppM = ievt%neppM; // #event in the current eventpage in this iteration - //printf( "%f\n", momenta[ipagM*npar*np4*neppM + ipar*np4*neppM + ip4*neppM + ieppM] ); + constexpr int neppM = mgOnGpu::neppM; // AOSOA layout: constant at compile-time + const int ipagM = ievt/neppM; // #event "M-page" + const int ieppM = ievt%neppM; // #event in the current event M-page + //printf( "%2d %2d %8d %8.3f\n", ipar, ip4, ievt, momenta[ipagM*npar*np4*neppM + ipar*np4*neppM + ip4*neppM + ieppM] ); return momenta[ipagM*npar*np4*neppM + ipar*np4*neppM + ip4*neppM + ieppM]; // AOSOA[ipagM][ipar][ip4][ieppM] //fptype (*momenta)[npar][np4][neppM] = (fptype (*)[npar][np4][neppM]) momenta; // cast to multiD array pointer (AOSOA) //return momenta[ipagM][ipar][ip4][ieppM]; // this seems ~1-2% faster in eemumu C++? } -#else - // Return by value: it seems a tiny bit faster than returning a reference (both for scalar and vector), not clear why + +#ifndef __CUDACC__ + // Return a SIMD vector of fptype's for neppV events (for the given particle, 4-momentum component and event "V-page") + // Return the vector by value: strictly speaking, this is only unavoidable for neppM=neppV (both being powers of 2), the momentum neppM-AOSOA is reinterpreted in terms of neppV-vectors: + // it could also be returned by reference, but no performance degradation is observed when returning by value inline - fptype_sv pIparIp4Ipag( const fptype_sv* momenta, // input: momenta as AOSOA[npagM][npar][4][neppM] + fptype_sv pIparIp4Ipag( const fptype* momenta, // input: momenta as AOSOA[npagM][npar][4][neppM] const int ipar, const int ip4, - const int ipagM ) + const int ipagV ) { - /* - //#ifndef MGONGPU_CPPSIMD - // TEMPORARY during epochX step3! Eventually remove this section (start) - // TEMPORARY during epochX step3! THERE IS NO SIMD YET: HENCE ipagM=ievt - // NB: this is needed for neppM>1 while neppV==1 (no SIMD) in eemumu.auto - // NB: this remains valid for neppM==1 with neppV==1 in eemumu (scalar) - return pIparIp4Ievt( momenta, ipar, ip4, ipagM ); - // TEMPORARY during epochX step3! Eventually remove this section (end) - //#else - */ - // NB: this assumes that neppV == neppM! - // NB: this is the same as "pIparIp4Ievt( momenta, ipar, ip4, ipagM )" for neppM==1 with neppV==1 - using mgOnGpu::np4; - using mgOnGpu::npar; - //printf( "%f\n", momenta[ipagM*npar*np4 + ipar*np4 + ip4] ); - return momenta[ipagM*npar*np4 + ipar*np4 + ip4]; // AOSOA[ipagM][ipar][ip4][ieppM] + const int ievt0 = ipagV*neppV; // virtual event V-page ipagV contains neppV events [ievt0...ievt0+neppV-1] +#ifdef MGONGPU_CPPSIMD + constexpr int neppM = mgOnGpu::neppM; // AOSOA layout: constant at compile-time + // Use c++17 "if constexpr": compile-time branching + if constexpr ( ( neppM >= neppV ) && ( neppM%neppV == 0 ) ) + { + constexpr bool useReinterpretCastIfPossible = true; // DEFAULT + //constexpr bool useReinterpretCastIfPossible = false; // FOR PERFORMANCE TESTS + constexpr bool skipAlignmentCheck = true; // DEFAULT (MAY SEGFAULT, NEEDS A SANITY CHECK ELSEWHERE!) + //constexpr bool skipAlignmentCheck = false; // SLOWER BUT SAFER + if constexpr ( useReinterpretCastIfPossible && skipAlignmentCheck ) + { + //static bool first=true; if( first ){ std::cout << "WARNING! skip alignment check" << std::endl; first=false; } // SLOWS DOWN... + // Fastest (4.92E6 in eemumu 512y) + // This requires alignment for momenta - segmentation fault otherwise! + return *reinterpret_cast( &( pIparIp4Ievt( momenta, ipar, ip4, ievt0 ) ) ); + } + else if ( useReinterpretCastIfPossible && ( (size_t)(momenta) % mgOnGpu::cppAlign == 0 ) ) + { + //static bool first=true; if( first ){ std::cout << "WARNING! alignment ok, use reinterpret cast" << std::endl; first=false; } // SLOWS DOWN... + // A bit (6%) slower (4.62E6 in eemumu 512y) because of the alignment check + // This explicitly checks alignment for momenta to avoid segmentation faults + return *reinterpret_cast( &( pIparIp4Ievt( momenta, ipar, ip4, ievt0 ) ) ); + } + else + { + //static bool first=true; if( first ){ std::cout << "WARNING! AOSOA but no reinterpret cast" << std::endl; first=false; } // SLOWS DOWN... + // A bit (2%) slower (4.86E6 in eemumu 512y) + // This does not require alignment for momenta, but it requires AOSOA with neppM>=neppV and neppM%neppV==0 + const fptype* out0 = &( pIparIp4Ievt( momenta, ipar, ip4, ievt0) ); +#if MGONGPU_CPPSIMD == 2 + return fptype_v{ *( out0 ), + *( out0+1 ) }; +#elif MGONGPU_CPPSIMD == 4 + return fptype_v{ *( out0 ), + *( out0+1 ), + *( out0+2 ), + *( out0+3 ) }; +#elif MGONGPU_CPPSIMD == 8 + return fptype_v{ *( out0 ), + *( out0+1 ), + *( out0+2 ), + *( out0+3 ), + *( out0+4 ), + *( out0+5 ), + *( out0+6 ), + *( out0+7 ) }; +#elif MGONGPU_CPPSIMD == 16 + return fptype_v{ *( out0 ), + *( out0+1 ), + *( out0+2 ), + *( out0+3 ), + *( out0+4 ), + *( out0+5 ), + *( out0+6 ), + *( out0+7 ), + *( out0+8 ), + *( out0+9 ), + *( out0+10 ), + *( out0+11 ), + *( out0+12 ), + *( out0+13 ), + *( out0+14 ), + *( out0+15 ) }; +#else +#warning Internal error? This code should never be reached! + // A bit (2%) slower (4.86E6 in eemumu 512y) + // This does not require alignment for momenta, but it requires AOSOA with neppM>=neppV and neppM%neppV==0 + fptype_v out; + for ( int ieppV=0; ieppV=neppV and neppM%neppV==0 (e.g. can be used with AOS neppM==1) +#if MGONGPU_CPPSIMD == 2 + return fptype_v{ pIparIp4Ievt( momenta, ipar, ip4, ievt0 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+1 ) }; +#elif MGONGPU_CPPSIMD == 4 + return fptype_v{ pIparIp4Ievt( momenta, ipar, ip4, ievt0 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+1 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+2 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+3 ) }; +#elif MGONGPU_CPPSIMD == 8 + return fptype_v{ pIparIp4Ievt( momenta, ipar, ip4, ievt0 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+1 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+2 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+3 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+4 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+5 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+6 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+7 ) }; +#elif MGONGPU_CPPSIMD == 16 + return fptype_v{ pIparIp4Ievt( momenta, ipar, ip4, ievt0 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+1 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+2 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+3 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+4 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+5 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+6 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+7 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+8 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+9 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+10 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+11 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+12 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+13 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+14 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+15 ) }; +#else +#warning Internal error? This code should never be reached! + // Much much (40%) slower (3.02E6 in eemumu 512y) + fptype_v out; + for ( int ieppV=0; ieppV 0) __device__ - void ipzxxx( const fptype_sv* momenta, + void ipzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -257,7 +368,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) __device__ - void imzxxx( const fptype_sv* momenta, + void imzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -306,7 +417,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PT > 0) __device__ - void ixzxxx( const fptype_sv* momenta, + void ixzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -367,7 +478,7 @@ namespace MG5_sm // Compute the output wavefunction vc[6] from the input momenta[npar*4*nevt] __device__ - void vxxxxx( const fptype_sv* momenta, + void vxxxxx( const fptype* momenta, const fptype vmass, // input: vector boson mass const int nhel, // input: -1, 0 (only if vmass!=0) or +1 (helicity of vector boson) const int nsv, // input: +1 (final) or -1 (initial) @@ -503,7 +614,7 @@ namespace MG5_sm // Compute the output wavefunction sc[3] from the input momenta[npar*4*nevt] __device__ - void sxxxxx( const fptype_sv* momenta, + void sxxxxx( const fptype* momenta, const fptype, // WARNING: input "smass" unused (missing in Fortran) - scalar boson mass const int, // WARNING: input "nhel" unused (missing in Fortran) - scalar has no helicity! const int nss, // input: +1 (final) or -1 (initial) @@ -544,7 +655,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] __device__ - void oxxxxx( const fptype_sv* momenta, + void oxxxxx( const fptype* momenta, const fptype fmass, // input: fermion mass const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -681,7 +792,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) __device__ - void opzxxx( const fptype_sv* momenta, + void opzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -730,7 +841,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) __device__ - void omzxxx( const fptype_sv* momenta, + void omzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -782,7 +893,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PT > 0) __device__ - void oxzxxx( const fptype_sv* momenta, + void oxzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) diff --git a/epochX/cudacpp/ee_mumu.auto/src/HelAmps_sm.h b/epochX/cudacpp/ee_mumu.auto/src/HelAmps_sm.h index 877ea1505d..7e2a127c98 100644 --- a/epochX/cudacpp/ee_mumu.auto/src/HelAmps_sm.h +++ b/epochX/cudacpp/ee_mumu.auto/src/HelAmps_sm.h @@ -28,7 +28,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] __device__ INLINE - void ixxxxx( const fptype_sv* momenta, + void ixxxxx( const fptype* momenta, const fptype fmass, // input: fermion mass const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -43,7 +43,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) __device__ INLINE - void ipzxxx( const fptype_sv* momenta, + void ipzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -58,7 +58,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) __device__ INLINE - void imzxxx( const fptype_sv* momenta, + void imzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -73,7 +73,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PT > 0) __device__ INLINE - void ixzxxx( const fptype_sv* momenta, + void ixzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -87,7 +87,7 @@ namespace MG5_sm // Compute the output wavefunction vc[6] from the input momenta[npar*4*nevt] __device__ INLINE - void vxxxxx( const fptype_sv* momenta, + void vxxxxx( const fptype* momenta, const fptype vmass, // input: vector boson mass const int nhel, // input: -1, 0 (only if vmass!=0) or +1 (helicity of vector boson) const int nsv, // input: +1 (final) or -1 (initial) @@ -101,7 +101,7 @@ namespace MG5_sm // Compute the output wavefunction sc[3] from the input momenta[npar*4*nevt] __device__ INLINE - void sxxxxx( const fptype_sv* momenta, + void sxxxxx( const fptype* momenta, const fptype, // WARNING: input "smass" unused (missing in Fortran) - scalar boson mass const int, // WARNING: input "nhel" unused (missing in Fortran) - scalar has no helicity! const int nss, // input: +1 (final) or -1 (initial) @@ -115,7 +115,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] __device__ INLINE - void oxxxxx( const fptype_sv* momenta, + void oxxxxx( const fptype* momenta, const fptype fmass, // input: fermion mass const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -130,7 +130,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) __device__ INLINE - void opzxxx( const fptype_sv* momenta, + void opzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -145,7 +145,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) __device__ INLINE - void omzxxx( const fptype_sv* momenta, + void omzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -159,7 +159,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] __device__ INLINE - void oxzxxx( const fptype_sv* momenta, + void oxzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) diff --git a/epochX/cudacpp/ee_mumu.auto/src/mgOnGpuConfig.h b/epochX/cudacpp/ee_mumu.auto/src/mgOnGpuConfig.h index 39b12d3515..3b639644a3 100644 --- a/epochX/cudacpp/ee_mumu.auto/src/mgOnGpuConfig.h +++ b/epochX/cudacpp/ee_mumu.auto/src/mgOnGpuConfig.h @@ -96,47 +96,79 @@ namespace mgOnGpu //const int ntpbMAX = 256; // AV Apr2021: why had I set this to 256? const int ntpbMAX = 1024; // NB: 512 is ok, but 1024 does fail with "too many resources requested for launch" - // Vector sizes for AOSOA memory layouts (GPU coalesced memory access, CPU SIMD vectorization) - // (these are all best kept as a compile-time constants: see issue #23) + // Alignment requirement for using reinterpret_cast with SIMD vectorized code + // (using reinterpret_cast with non aligned memory may lead to segmentation faults!) +#ifndef __CUDACC__ + constexpr int cppAlign = 64; // alignment requirement for SIMD vectorization (64-byte i.e. 512-bit) +#endif - // Number of Events Per Page in the momenta AOSOA memory layout + // C++ SIMD vectorization width (this will be used to set neppV) #ifdef __CUDACC__ #undef MGONGPU_CPPSIMD - // ----------------------------------------------------------------------------------------- - // --- GPUs: neppM must be a power of 2 times the number of fptype's in a 32-byte cacheline - // --- This is relevant to ensure coalesced access to momenta in global memory - // --- Note that neppR is hardcoded and may differ from neppM and neppV on some platforms - // ----------------------------------------------------------------------------------------- - //const int neppM = 64/sizeof(fptype); // 2x 32-byte GPU cache lines: 8 (DOUBLE) or 16 (FLOAT) - const int neppM = 32/sizeof(fptype); // (DEFAULT) 32-byte GPU cache line: 4 (DOUBLE) or 8 (FLOAT) - //const int neppM = 1; // *** NB: this is equivalent to AOS *** - //const int neppM = 32; // older default #else - // ----------------------------------------------------------------------------------------- - // --- CPUs: neppM must be exactly equal to the number of fptype's in a vector register - // --- [DEFAULT is 256-width AVX512 aka "512y" on CPUs, 32-byte as GPUs, faster than AVX2] - // --- The logic of the code requires the size neppV of fptype_v to be equal to neppM - // --- Note that neppR is hardcoded and may differ from neppM and neppV on some platforms - // ----------------------------------------------------------------------------------------- #if defined __AVX512VL__ -#define MGONGPU_CPPSIMD 1 #ifdef MGONGPU_PVW512 - const int neppM = 64/sizeof(fptype); // "512z" AVX512 with 512 width (512-bit ie 64-byte): 8 (DOUBLE) or 16 (FLOAT) + // "512z" AVX512 with 512 width (512-bit ie 64-byte): 8 (DOUBLE) or 16 (FLOAT) +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 8 #else - const int neppM = 32/sizeof(fptype); // "512y" AVX512 with 256 width (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [gcc DEFAULT] +#define MGONGPU_CPPSIMD 16 +#endif +#else + // "512y" AVX512 with 256 width (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [gcc DEFAULT] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 4 +#else +#define MGONGPU_CPPSIMD 8 +#endif #endif #elif defined __AVX2__ -#define MGONGPU_CPPSIMD 1 - const int neppM = 32/sizeof(fptype); // "avx2" AVX2 (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [clang DEFAULT] + // "avx2" AVX2 (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [clang DEFAULT] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 4 +#else +#define MGONGPU_CPPSIMD 8 +#endif #elif defined __SSE4_2__ -#define MGONGPU_CPPSIMD 1 - const int neppM = 16/sizeof(fptype); // "sse4" SSE4.2 (128-bit ie 16-byte): 2 (DOUBLE) or 4 (FLOAT) + // "sse4" SSE4.2 (128-bit ie 16-byte): 2 (DOUBLE) or 4 (FLOAT) +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 2 #else +#define MGONGPU_CPPSIMD 4 +#endif +#else + // "none" i.e. no SIMD (*** NB: this is equivalent to AOS ***) #undef MGONGPU_CPPSIMD - const int neppM = 1; // "none" i.e. no SIMD (*** NB: this is equivalent to AOS ***) #endif #endif + // Number of Events Per Page in the momenta AOSOA memory layout +#ifdef __CUDACC__ + // ----------------------------------------------------------------------------------------------- + // --- GPUs: neppM is best set to a power of 2 times the number of fptype's in a 32-byte cacheline + // --- This is relevant to ensure coalesced access to momenta in global memory + // --- Note that neppR is hardcoded and may differ from neppM and neppV on some platforms + // ----------------------------------------------------------------------------------------------- + //const int neppM = 64/sizeof(fptype); // 2x 32-byte GPU cache lines (512 bits): 8 (DOUBLE) or 16 (FLOAT) + const int neppM = 32/sizeof(fptype); // (DEFAULT) 32-byte GPU cache line (256 bits): 4 (DOUBLE) or 8 (FLOAT) + //const int neppM = 1; // *** NB: this is equivalent to AOS *** +#else + // ----------------------------------------------------------------------------------------------- + // --- CPUs: neppM is best set equal to the number of fptype's (neppV) in a vector register + // --- This is relevant to ensure faster access to momenta from C++ memory cache lines + // --- However, neppM is now decoupled from neppV (issue #176) and can be separately hardcoded + // --- In practice, neppR, neppM and neppV could now (in principle) all be different + // ----------------------------------------------------------------------------------------------- +#ifdef MGONGPU_CPPSIMD + const int neppM = MGONGPU_CPPSIMD; // (DEFAULT) neppM=neppV for optimal performance +#else + const int neppM = 1; // (DEFAULT) neppM=neppV for optimal performance (NB: this is equivalent to AOS) +#endif + //const int neppM = 64/sizeof(fptype); // maximum CPU vector width (512 bits): 8 (DOUBLE) or 16 (FLOAT) + //const int neppM = 32/sizeof(fptype); // lower CPU vector width (256 bits): 4 (DOUBLE) or 8 (FLOAT) + //const int neppM = 1; // *** NB: this is equivalent to AOS *** +#endif + // Number of Events Per Page in the random number AOSOA memory layout // *** NB Different values of neppR lead to different physics results: the *** // *** same 1d array is generated, but it is interpreted in different ways *** diff --git a/epochX/cudacpp/ee_mumu.auto/src/mgOnGpuVectors.h b/epochX/cudacpp/ee_mumu.auto/src/mgOnGpuVectors.h index e1da365663..4e975b2ca1 100644 --- a/epochX/cudacpp/ee_mumu.auto/src/mgOnGpuVectors.h +++ b/epochX/cudacpp/ee_mumu.auto/src/mgOnGpuVectors.h @@ -13,9 +13,13 @@ namespace mgOnGpu { + #ifdef MGONGPU_CPPSIMD - const int neppV = neppM; + const int neppV = MGONGPU_CPPSIMD; + + // SANITY CHECK: cppAlign must be a multiple of neppV * sizeof(fptype) + static_assert( mgOnGpu::cppAlign % ( neppV * sizeof(fptype) ) == 0 ); // --- Type definition (using vector compiler extensions: need -march=...) #ifdef __clang__ // https://clang.llvm.org/docs/LanguageExtensions.html#vectors-and-extended-vectors @@ -96,7 +100,7 @@ namespace mgOnGpu #else // MGONGPU_CPPSIMD not defined - const int neppV = 1; // Note: also neppM is equal to 1 + const int neppV = 1; #endif } @@ -114,7 +118,7 @@ using mgOnGpu::bool_v; inline std::ostream& operator<<( std::ostream& out, const bool_v& v ) { out << "{ " << v[0]; - for ( int i=1; i #include @@ -22,7 +22,7 @@ namespace rambo2toNm0 // [NB: the output buffer includes both initial and final momenta, but only initial momenta are filled in] __global__ void getMomentaInitial( const fptype energy, // input: energy - fptype_sv momenta1d[] // output: momenta as AOSOA[npagM][npar][4][neppM] + fptype momenta1d[] // output: momenta as AOSOA[npagM][npar][4][neppM] #ifndef __CUDACC__ , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) #endif @@ -63,7 +63,7 @@ namespace rambo2toNm0 __global__ void getMomentaFinal( const fptype energy, // input: energy const fptype rnarray1d[], // input: random numbers in [0,1] as AOSOA[npagR][nparf][4][neppR] - fptype_sv momenta1d[], // output: momenta as AOSOA[npagM][npar][4][neppM] + fptype momenta1d[], // output: momenta as AOSOA[npagM][npar][4][neppM] fptype wgts[] // output: weights[nevt] #ifndef __CUDACC__ , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) diff --git a/epochX/cudacpp/ee_mumu.auto/src/rambo.h b/epochX/cudacpp/ee_mumu.auto/src/rambo.h index 030526759f..9d9e876fe3 100644 --- a/epochX/cudacpp/ee_mumu.auto/src/rambo.h +++ b/epochX/cudacpp/ee_mumu.auto/src/rambo.h @@ -3,7 +3,7 @@ #include "mgOnGpuConfig.h" #include "mgOnGpuTypes.h" -#include "mgOnGpuVectors.h" +//#include "mgOnGpuVectors.h" #include @@ -47,7 +47,7 @@ namespace rambo2toNm0 // [NB: the output buffer includes both initial and final momenta, but only initial momenta are filled in] __global__ void getMomentaInitial( const fptype energy, // input: energy - fptype_sv momenta1d[] // output: momenta as AOSOA[npagM][npar][4][neppM] + fptype momenta1d[] // output: momenta as AOSOA[npagM][npar][4][neppM] #ifndef __CUDACC__ , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) #endif @@ -60,7 +60,7 @@ namespace rambo2toNm0 __global__ void getMomentaFinal( const fptype energy, // input: energy const fptype rnarray1d[], // input: random numbers in [0,1] as AOSOA[npagR][nparf][4][neppR] - fptype_sv momenta1d[], // output: momenta as AOSOA[npagM][npar][4][neppM] + fptype momenta1d[], // output: momenta as AOSOA[npagM][npar][4][neppM] fptype wgts[] // output: weights[nevt] #ifndef __CUDACC__ , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) From 25daaadffcb4a9a1f0424845d216ec41ebf4e4e5 Mon Sep 17 00:00:00 2001 From: Andrea Valassi Date: Wed, 1 Dec 2021 10:12:09 +0100 Subject: [PATCH 12/34] [klas5X] rerun all perf tests for eemumu, including float and inline - few minor changes, mainly for inline (but I do not care about inline as it is not really relevant for ggttgg...) --- .../log_eemumu_manu_d_inl0.txt | 74 ++++++------ .../log_eemumu_manu_d_inl1.txt | 106 +++++++++--------- .../log_eemumu_manu_f_inl0.txt | 106 +++++++++--------- .../log_eemumu_manu_f_inl1.txt | 106 +++++++++--------- 4 files changed, 196 insertions(+), 196 deletions(-) diff --git a/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_d_inl0.txt b/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_d_inl0.txt index 34dde48eb0..24715a4b4e 100644 --- a/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_d_inl0.txt +++ b/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_d_inl0.txt @@ -40,20 +40,20 @@ RNDGEN=curdev Building in BUILDDIR=build.512z_d_inl0 for tag=512z_d_inl0_curdev (USEBUILDDIR is set = 1) make: Nothing to be done for `all.512z_d_inl0_curdev'. -DATE: 2021-11-30_18:26:43 +DATE: 2021-12-01_10:07:10 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl0/gcheck.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 6.429697e+08 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.106342e+09 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 6.534485e+08 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.116555e+09 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 0.707901 sec - 373,331,879 cycles:u # 0.396 GHz - 693,060,121 instructions:u # 1.86 insn per cycle - 1.006807629 seconds time elapsed +TOTAL : 0.709466 sec + 371,909,202 cycles:u # 0.399 GHz + 697,385,836 instructions:u # 1.88 insn per cycle + 0.996526017 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 130 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... @@ -62,13 +62,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.340565e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.340565e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.337315e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.337315e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 7.024663 sec - 18,630,578,672 cycles:u # 2.651 GHz - 48,224,156,791 instructions:u # 2.59 insn per cycle - 7.032416618 seconds time elapsed +TOTAL : 7.047599 sec + 18,666,706,719 cycles:u # 2.647 GHz + 48,224,158,012 instructions:u # 2.58 insn per cycle + 7.055088398 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 621) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl0/runTest.exe @@ -78,13 +78,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[2] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 2.566794e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 2.566794e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 2.565767e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 2.565767e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 4.832176 sec - 12,735,236,411 cycles:u # 2.634 GHz - 29,850,563,842 instructions:u # 2.34 insn per cycle - 4.840327392 seconds time elapsed +TOTAL : 4.823406 sec + 12,729,876,419 cycles:u # 2.637 GHz + 29,850,563,434 instructions:u # 2.34 insn per cycle + 4.831352437 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 3297) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_d_inl0/runTest.exe @@ -94,13 +94,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 4.559694e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.559694e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.556887e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.556887e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 3.662016 sec - 9,085,677,498 cycles:u # 2.478 GHz - 16,644,667,698 instructions:u # 1.83 insn per cycle - 3.669537861 seconds time elapsed +TOTAL : 3.663924 sec + 9,087,880,283 cycles:u # 2.478 GHz + 16,644,667,763 instructions:u # 1.83 insn per cycle + 3.671637372 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2831) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_d_inl0/runTest.exe @@ -110,13 +110,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 4.934023e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.934023e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.921532e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.921532e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 3.571025 sec - 8,891,334,309 cycles:u # 2.487 GHz - 16,531,423,280 instructions:u # 1.86 insn per cycle - 3.578914335 seconds time elapsed +TOTAL : 3.602738 sec + 8,968,063,737 cycles:u # 2.486 GHz + 16,531,423,183 instructions:u # 1.84 insn per cycle + 3.610374997 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2728) (512y: 52) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_d_inl0/runTest.exe @@ -126,13 +126,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 3.817528e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.817528e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 3.828049e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.828049e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 3.968037 sec - 8,660,592,511 cycles:u # 2.180 GHz - 13,280,272,764 instructions:u # 1.53 insn per cycle - 3.975661225 seconds time elapsed +TOTAL : 3.958997 sec + 8,638,325,570 cycles:u # 2.180 GHz + 13,280,272,453 instructions:u # 1.54 insn per cycle + 3.966662948 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1319) (512y: 66) (512z: 2147) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_d_inl0/runTest.exe diff --git a/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_d_inl1.txt b/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_d_inl1.txt index 8bc12fb4f4..17c1e59cc3 100644 --- a/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_d_inl1.txt +++ b/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_d_inl1.txt @@ -1,5 +1,5 @@ -/afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum +/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum OMPFLAGS= AVX=none FPTYPE=d @@ -40,102 +40,102 @@ RNDGEN=curdev Building in BUILDDIR=build.512z_d_inl1 for tag=512z_d_inl1_curdev (USEBUILDDIR is set = 1) make: Nothing to be done for `all.512z_d_inl1_curdev'. -DATE: 2021-10-30_18:38:26 +DATE: 2021-12-01_10:07:46 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl1/gcheck.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl1/gcheck.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 6.533413e+08 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.111850e+09 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 6.494723e+08 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.109138e+09 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 1.014189 sec - 1,012,862,888 cycles:u # 0.840 GHz - 2,001,256,749 instructions:u # 1.98 insn per cycle - 1.307143919 seconds time elapsed +TOTAL : 1.226259 sec + 997,952,337 cycles:u # 0.839 GHz + 1,981,968,331 instructions:u # 1.99 insn per cycle + 1.518449844 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 130 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl1/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl1/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 4.733180e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.733180e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.736845e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.736845e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 3.719327 sec - 9,704,160,009 cycles:u # 2.602 GHz - 17,889,581,677 instructions:u # 1.84 insn per cycle - 3.732898223 seconds time elapsed +TOTAL : 3.677158 sec + 9,670,909,975 cycles:u # 2.627 GHz + 17,889,570,187 instructions:u # 1.85 insn per cycle + 3.684775451 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 2595) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_d_inl1/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_d_inl1/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[2] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 9.310040e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 9.310040e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 9.455274e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 9.455274e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 3.071982 sec - 8,002,098,898 cycles:u # 2.596 GHz - 14,177,608,599 instructions:u # 1.77 insn per cycle - 3.085850687 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 476) (avx2: 0) (512y: 0) (512z: 0) +TOTAL : 3.046095 sec + 7,995,654,626 cycles:u # 2.622 GHz + 13,964,472,664 instructions:u # 1.75 insn per cycle + 3.053929213 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 432) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_d_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_d_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_d_inl1/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_d_inl1/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.312561e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.312561e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.303904e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.303904e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 2.795090 sec - 7,068,426,417 cycles:u # 2.520 GHz - 11,097,921,043 instructions:u # 1.57 insn per cycle - 2.809351306 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 426) (512y: 0) (512z: 0) +TOTAL : 2.773556 sec + 7,073,285,248 cycles:u # 2.547 GHz + 11,086,106,492 instructions:u # 1.57 insn per cycle + 2.781104378 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 461) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_d_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_d_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_d_inl1/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_d_inl1/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.421422e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.421422e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.383099e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.383099e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 2.782304 sec - 7,056,747,465 cycles:u # 2.526 GHz - 10,931,197,726 instructions:u # 1.55 insn per cycle - 2.796126198 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 371) (512y: 8) (512z: 0) +TOTAL : 2.767352 sec + 7,074,744,001 cycles:u # 2.552 GHz + 10,919,382,397 instructions:u # 1.54 insn per cycle + 2.775205710 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 412) (512y: 1) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_d_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_d_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_d_inl1/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_d_inl1/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 9.951438e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 9.951438e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 9.344111e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 9.344111e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 2.978139 sec - 7,064,137,452 cycles:u # 2.364 GHz - 10,423,152,716 instructions:u # 1.48 insn per cycle - 2.991902693 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 130) (512y: 4) (512z: 264) +TOTAL : 3.015544 sec + 7,170,726,069 cycles:u # 2.377 GHz + 10,478,969,947 instructions:u # 1.46 insn per cycle + 3.023095729 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 193) (512y: 2) (512z: 230) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_d_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_d_inl1/runTest.exe [ PASSED ] 3 tests. ========================================================================= diff --git a/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_f_inl0.txt b/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_f_inl0.txt index cfd6517a1d..30e3d856fb 100644 --- a/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_f_inl0.txt +++ b/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_f_inl0.txt @@ -1,5 +1,5 @@ -/afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum +/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum OMPFLAGS= AVX=none FPTYPE=f @@ -40,102 +40,102 @@ RNDGEN=curdev Building in BUILDDIR=build.512z_f_inl0 for tag=512z_f_inl0_curdev (USEBUILDDIR is set = 1) make: Nothing to be done for `all.512z_f_inl0_curdev'. -DATE: 2021-10-30_18:38:55 +DATE: 2021-12-01_10:08:12 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl0/gcheck.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl0/gcheck.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=2, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 1.516650e+09 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.225614e+09 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.567154e+09 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.236935e+09 ) sec^-1 MeanMatrixElemValue = ( 1.371686e-02 +- 3.270219e-06 ) GeV^0 -TOTAL : 0.855293 sec - 727,164,355 cycles:u # 0.702 GHz - 1,500,898,923 instructions:u # 2.06 insn per cycle - 1.142230867 seconds time elapsed +TOTAL : 0.842427 sec + 716,989,672 cycles:u # 0.704 GHz + 1,484,422,831 instructions:u # 2.07 insn per cycle + 1.126361376 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 56 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl0/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl0/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=6, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.206818e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.206818e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.224589e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.224589e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371707e-02 +- 3.270376e-06 ) GeV^0 -TOTAL : 7.142694 sec - 18,974,115,268 cycles:u # 2.656 GHz - 47,311,632,298 instructions:u # 2.49 insn per cycle - 7.150468071 seconds time elapsed +TOTAL : 7.053663 sec + 18,764,093,689 cycles:u # 2.659 GHz + 47,311,642,026 instructions:u # 2.52 insn per cycle + 7.060851302 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 559) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl0/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_f_inl0/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_f_inl0/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=6, zero=0) Internal loops fptype_sv = VECTOR[4] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 4.462560e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.462560e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.573579e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.573579e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270375e-06 ) GeV^0 -TOTAL : 3.345853 sec - 8,849,012,810 cycles:u # 2.641 GHz - 19,646,249,505 instructions:u # 2.22 insn per cycle - 3.353925880 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 3997) (avx2: 0) (512y: 0) (512z: 0) +TOTAL : 3.344023 sec + 8,846,286,898 cycles:u # 2.642 GHz + 19,615,196,579 instructions:u # 2.22 insn per cycle + 3.351351467 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 3969) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_f_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_f_inl0/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_f_inl0/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_f_inl0/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=5, zero=0) Internal loops fptype_sv = VECTOR[8] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 8.238606e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 8.238606e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 8.181232e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 8.181232e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371705e-02 +- 3.270339e-06 ) GeV^0 -TOTAL : 2.665525 sec - 6,724,173,249 cycles:u # 2.518 GHz - 12,573,014,170 instructions:u # 1.87 insn per cycle - 2.673669036 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 3133) (512y: 0) (512z: 0) +TOTAL : 2.698787 sec + 6,823,046,265 cycles:u # 2.524 GHz + 12,524,650,277 instructions:u # 1.84 insn per cycle + 2.705979783 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 3175) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_f_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_f_inl0/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_f_inl0/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_f_inl0/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=5, zero=0) Internal loops fptype_sv = VECTOR[8] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 8.927598e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 8.927598e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 8.961058e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 8.961058e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371705e-02 +- 3.270339e-06 ) GeV^0 -TOTAL : 2.607120 sec - 6,595,157,277 cycles:u # 2.525 GHz - 12,525,829,761 instructions:u # 1.90 insn per cycle - 2.614878683 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 3023) (512y: 28) (512z: 0) +TOTAL : 2.587116 sec + 6,548,070,094 cycles:u # 2.527 GHz + 12,477,465,603 instructions:u # 1.91 insn per cycle + 2.594729483 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 3069) (512y: 26) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_f_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_f_inl0/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_f_inl0/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_f_inl0/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=5, zero=0) Internal loops fptype_sv = VECTOR[16] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 7.393500e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 7.393500e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 7.493309e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 7.493309e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371705e-02 +- 3.270340e-06 ) GeV^0 -TOTAL : 2.750985 sec - 6,313,615,433 cycles:u # 2.291 GHz - 10,854,649,706 instructions:u # 1.72 insn per cycle - 2.758962696 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1674) (512y: 13) (512z: 2271) +TOTAL : 2.727509 sec + 6,281,035,658 cycles:u # 2.300 GHz + 10,798,421,531 instructions:u # 1.72 insn per cycle + 2.734747652 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1772) (512y: 12) (512z: 2256) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_f_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_f_inl0/runTest.exe [ PASSED ] 3 tests. ========================================================================= diff --git a/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_f_inl1.txt b/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_f_inl1.txt index af6dfdea32..adffd368b3 100644 --- a/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_f_inl1.txt +++ b/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_f_inl1.txt @@ -1,5 +1,5 @@ -/afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum +/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum OMPFLAGS= AVX=none FPTYPE=f @@ -40,102 +40,102 @@ RNDGEN=curdev Building in BUILDDIR=build.512z_f_inl1 for tag=512z_f_inl1_curdev (USEBUILDDIR is set = 1) make: Nothing to be done for `all.512z_f_inl1_curdev'. -DATE: 2021-10-30_18:39:46 +DATE: 2021-12-01_10:08:43 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl1/gcheck.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl1/gcheck.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=2, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 1.430147e+09 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.241660e+09 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.577348e+09 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.256680e+09 ) sec^-1 MeanMatrixElemValue = ( 1.371686e-02 +- 3.270219e-06 ) GeV^0 -TOTAL : 0.929019 sec - 731,390,576 cycles:u # 0.702 GHz - 1,497,204,981 instructions:u # 2.05 insn per cycle - 1.219085654 seconds time elapsed +TOTAL : 0.907275 sec + 718,084,580 cycles:u # 0.706 GHz + 1,482,928,433 instructions:u # 2.07 insn per cycle + 1.190328356 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 56 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl1/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl1/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=6, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.221707e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.221707e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.203281e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.203281e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270375e-06 ) GeV^0 -TOTAL : 2.440777 sec - 6,434,889,296 cycles:u # 2.631 GHz - 12,595,058,563 instructions:u # 1.96 insn per cycle - 2.448468922 seconds time elapsed +TOTAL : 2.431255 sec + 6,417,013,841 cycles:u # 2.634 GHz + 12,595,066,654 instructions:u # 1.96 insn per cycle + 2.438826820 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 796) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_f_inl1/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_f_inl1/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=6, zero=0) Internal loops fptype_sv = VECTOR[4] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 2.000845e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 2.000845e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.923450e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.923450e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270375e-06 ) GeV^0 -TOTAL : 2.254327 sec - 5,941,500,593 cycles:u # 2.630 GHz - 11,332,004,392 instructions:u # 1.91 insn per cycle - 2.262004870 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 503) (avx2: 0) (512y: 0) (512z: 0) +TOTAL : 2.297739 sec + 6,058,657,245 cycles:u # 2.632 GHz + 11,389,032,042 instructions:u # 1.88 insn per cycle + 2.304986309 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 508) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_f_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_f_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_f_inl1/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_f_inl1/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=5, zero=0) Internal loops fptype_sv = VECTOR[8] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 2.942985e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 2.942985e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 2.894889e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 2.894889e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371705e-02 +- 3.270339e-06 ) GeV^0 -TOTAL : 2.131788 sec - 5,510,571,346 cycles:u # 2.583 GHz - 9,702,504,319 instructions:u # 1.76 insn per cycle - 2.139640046 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 481) (512y: 0) (512z: 0) +TOTAL : 2.105405 sec + 5,454,622,456 cycles:u # 2.585 GHz + 9,655,713,280 instructions:u # 1.77 insn per cycle + 2.113229620 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 533) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_f_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_f_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_f_inl1/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_f_inl1/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=5, zero=0) Internal loops fptype_sv = VECTOR[8] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 3.054135e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.054135e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 3.028386e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.028386e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371705e-02 +- 3.270339e-06 ) GeV^0 -TOTAL : 2.105307 sec - 5,459,860,062 cycles:u # 2.588 GHz - 9,617,570,649 instructions:u # 1.76 insn per cycle - 2.112973584 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 440) (512y: 2) (512z: 0) +TOTAL : 2.093570 sec + 5,428,962,159 cycles:u # 2.588 GHz + 9,583,362,532 instructions:u # 1.77 insn per cycle + 2.101081895 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 497) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_f_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_f_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_f_inl1/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_f_inl1/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=5, zero=0) Internal loops fptype_sv = VECTOR[16] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 2.146895e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 2.146895e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 2.059238e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 2.059238e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371705e-02 +- 3.270340e-06 ) GeV^0 -TOTAL : 2.188251 sec - 5,431,722,356 cycles:u # 2.478 GHz - 9,341,535,791 instructions:u # 1.72 insn per cycle - 2.196084347 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 207) (512y: 0) (512z: 286) +TOTAL : 2.192812 sec + 5,434,991,260 cycles:u # 2.474 GHz + 9,332,494,253 instructions:u # 1.72 insn per cycle + 2.200097462 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 293) (512y: 0) (512z: 251) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_f_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_f_inl1/runTest.exe [ PASSED ] 3 tests. ========================================================================= From 2392f5e034a78a893b6bd6b5feec17edbfb5f2fb Mon Sep 17 00:00:00 2001 From: Andrea Valassi Date: Wed, 1 Dec 2021 10:40:59 +0100 Subject: [PATCH 13/34] [klas5X] add copyLogs script --- epochX/cudacpp/tput/copyLogs.sh | 61 +++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100755 epochX/cudacpp/tput/copyLogs.sh diff --git a/epochX/cudacpp/tput/copyLogs.sh b/epochX/cudacpp/tput/copyLogs.sh new file mode 100755 index 0000000000..210f38b311 --- /dev/null +++ b/epochX/cudacpp/tput/copyLogs.sh @@ -0,0 +1,61 @@ +#/bin/bash + +direction= +eemumu=0 +ggtt=0 +ggttgg=0 + +function usage() +{ + echo "Usage: $0 " + exit 1 +} + +while [ "$1" != "" ]; do + if [ "$1" == "-a2m" ] || [ "$1" == "-m2a" ]; then + if [ "$direction" != "" ] && [ "$direction" != "$1" ]; then echo "ERROR! Options $direction and $1 are incompatible"; usage; fi + direction=$1 + shift + elif [ "$1" == "-eemumu" ]; then + eemumu=1 + shift + ###elif [ "$1" == "-ggtt" ]; then + ### ggtt=1 + ### shift + elif [ "$1" == "-ggttgg" ]; then + ggttgg=1 + shift + else + usage + fi +done + +# Check that a direction has been selected +if [ "${direction}" == "-a2m" ]; then + src=auto; dst=manu +elif [ "${direction}" == "-m2a" ]; then + src=manu; dst=auto +else + usage +fi + +# Check that at least one process has been selected +processes= +if [ "${ggttgg}" == "1" ]; then processes="ggttgg $processes"; fi +###if [ "${ggtt}" == "1" ]; then processes="ggtt $processes"; fi +if [ "${eemumu}" == "1" ]; then processes="eemumu $processes"; fi +if [ "${processes}" == "" ]; then usage; fi + +echo "direction: ${direction}" +echo "processes: ${processes}" + +cd $(dirname $0) +for proc in ${processes}; do + echo "------------------------------------------------------------------" + for fs in logs_${proc}_${src}/log_${proc}_${src}_*; do + fd=$(basename $fs) + fd=logs_${proc}_${dst}/log_${proc}_${dst}_${fd#log_${proc}_${src}_} + echo \cp -dpr ${fs} ${fd} + \cp -dpr ${fs} ${fd} + done +done From 41dd9b74974bbf70d930e22a53a9d7577bd3262d Mon Sep 17 00:00:00 2001 From: Andrea Valassi Date: Wed, 1 Dec 2021 10:41:20 +0100 Subject: [PATCH 14/34] [klas5X] copy eemumu logs manu to auto (./copyLogs.sh -eemumu -m2a) --- .../log_eemumu_auto_d_inl0.txt | 106 +++++++++--------- .../log_eemumu_auto_d_inl1.txt | 106 +++++++++--------- .../log_eemumu_auto_f_inl0.txt | 106 +++++++++--------- .../log_eemumu_auto_f_inl1.txt | 106 +++++++++--------- 4 files changed, 212 insertions(+), 212 deletions(-) diff --git a/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_d_inl0.txt b/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_d_inl0.txt index 02a0058d89..24715a4b4e 100644 --- a/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_d_inl0.txt +++ b/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_d_inl0.txt @@ -1,5 +1,5 @@ -/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum +/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum OMPFLAGS= AVX=none FPTYPE=d @@ -40,102 +40,102 @@ RNDGEN=curdev Building in BUILDDIR=build.512z_d_inl0 for tag=512z_d_inl0_curdev (USEBUILDDIR is set = 1) make: Nothing to be done for `all.512z_d_inl0_curdev'. -DATE: 2021-11-16_14:24:33 +DATE: 2021-12-01_10:07:10 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= -runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl0/gcheck.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl0/gcheck.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 6.537105e+08 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.113819e+09 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 6.534485e+08 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.116555e+09 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 1.213842 sec - 727,998,694 cycles:u # 1.304 GHz - 1,430,713,650 instructions:u # 1.97 insn per cycle - 1.297817889 seconds time elapsed +TOTAL : 0.709466 sec + 371,909,202 cycles:u # 0.399 GHz + 697,385,836 instructions:u # 1.88 insn per cycle + 0.996526017 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 130 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... ========================================================================= -runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl0/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl0/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.316746e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.316746e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.337315e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.337315e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 7.134313 sec - 18,861,593,219 cycles:u # 2.647 GHz - 48,224,155,889 instructions:u # 2.56 insn per cycle - 7.142099106 seconds time elapsed +TOTAL : 7.047599 sec + 18,666,706,719 cycles:u # 2.647 GHz + 48,224,158,012 instructions:u # 2.58 insn per cycle + 7.055088398 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 621) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl0/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_d_inl0/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_d_inl0/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[2] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 2.531865e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 2.531865e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 2.565767e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 2.565767e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 4.848749 sec - 12,799,216,075 cycles:u # 2.637 GHz - 29,912,692,048 instructions:u # 2.34 insn per cycle - 4.856433199 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 3327) (avx2: 0) (512y: 0) (512z: 0) +TOTAL : 4.823406 sec + 12,729,876,419 cycles:u # 2.637 GHz + 29,850,563,434 instructions:u # 2.34 insn per cycle + 4.831352437 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 3297) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_d_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_d_inl0/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_d_inl0/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_d_inl0/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 4.505068e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.505068e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.556887e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.556887e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 3.697928 sec - 9,156,085,532 cycles:u # 2.476 GHz - 16,678,491,383 instructions:u # 1.82 insn per cycle - 3.705493481 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2806) (512y: 0) (512z: 0) +TOTAL : 3.663924 sec + 9,087,880,283 cycles:u # 2.478 GHz + 16,644,667,763 instructions:u # 1.83 insn per cycle + 3.671637372 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2831) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_d_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_d_inl0/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_d_inl0/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_d_inl0/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 4.955442e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.955442e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.921532e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.921532e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 3.574878 sec - 8,898,506,821 cycles:u # 2.487 GHz - 16,565,247,110 instructions:u # 1.86 insn per cycle - 3.582423532 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2703) (512y: 59) (512z: 0) +TOTAL : 3.602738 sec + 8,968,063,737 cycles:u # 2.486 GHz + 16,531,423,183 instructions:u # 1.84 insn per cycle + 3.610374997 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2728) (512y: 52) (512z: 0) ------------------------------------------------------------------------- -runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_d_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_d_inl0/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_d_inl0/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_d_inl0/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 3.683001e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.683001e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 3.828049e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.828049e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 4.060166 sec - 8,725,598,766 cycles:u # 2.149 GHz - 13,329,824,976 instructions:u # 1.53 insn per cycle - 4.067742325 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1251) (512y: 68) (512z: 2163) +TOTAL : 3.958997 sec + 8,638,325,570 cycles:u # 2.180 GHz + 13,280,272,453 instructions:u # 1.54 insn per cycle + 3.966662948 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1319) (512y: 66) (512z: 2147) ------------------------------------------------------------------------- -runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_d_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_d_inl0/runTest.exe [ PASSED ] 3 tests. ========================================================================= diff --git a/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_d_inl1.txt b/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_d_inl1.txt index 8bc12fb4f4..17c1e59cc3 100644 --- a/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_d_inl1.txt +++ b/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_d_inl1.txt @@ -1,5 +1,5 @@ -/afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum +/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum OMPFLAGS= AVX=none FPTYPE=d @@ -40,102 +40,102 @@ RNDGEN=curdev Building in BUILDDIR=build.512z_d_inl1 for tag=512z_d_inl1_curdev (USEBUILDDIR is set = 1) make: Nothing to be done for `all.512z_d_inl1_curdev'. -DATE: 2021-10-30_18:38:26 +DATE: 2021-12-01_10:07:46 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl1/gcheck.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl1/gcheck.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 6.533413e+08 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.111850e+09 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 6.494723e+08 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.109138e+09 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 1.014189 sec - 1,012,862,888 cycles:u # 0.840 GHz - 2,001,256,749 instructions:u # 1.98 insn per cycle - 1.307143919 seconds time elapsed +TOTAL : 1.226259 sec + 997,952,337 cycles:u # 0.839 GHz + 1,981,968,331 instructions:u # 1.99 insn per cycle + 1.518449844 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 130 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl1/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl1/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 4.733180e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.733180e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.736845e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.736845e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 3.719327 sec - 9,704,160,009 cycles:u # 2.602 GHz - 17,889,581,677 instructions:u # 1.84 insn per cycle - 3.732898223 seconds time elapsed +TOTAL : 3.677158 sec + 9,670,909,975 cycles:u # 2.627 GHz + 17,889,570,187 instructions:u # 1.85 insn per cycle + 3.684775451 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 2595) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_d_inl1/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_d_inl1/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[2] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 9.310040e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 9.310040e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 9.455274e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 9.455274e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 3.071982 sec - 8,002,098,898 cycles:u # 2.596 GHz - 14,177,608,599 instructions:u # 1.77 insn per cycle - 3.085850687 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 476) (avx2: 0) (512y: 0) (512z: 0) +TOTAL : 3.046095 sec + 7,995,654,626 cycles:u # 2.622 GHz + 13,964,472,664 instructions:u # 1.75 insn per cycle + 3.053929213 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 432) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_d_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_d_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_d_inl1/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_d_inl1/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.312561e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.312561e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.303904e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.303904e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 2.795090 sec - 7,068,426,417 cycles:u # 2.520 GHz - 11,097,921,043 instructions:u # 1.57 insn per cycle - 2.809351306 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 426) (512y: 0) (512z: 0) +TOTAL : 2.773556 sec + 7,073,285,248 cycles:u # 2.547 GHz + 11,086,106,492 instructions:u # 1.57 insn per cycle + 2.781104378 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 461) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_d_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_d_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_d_inl1/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_d_inl1/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.421422e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.421422e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.383099e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.383099e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 2.782304 sec - 7,056,747,465 cycles:u # 2.526 GHz - 10,931,197,726 instructions:u # 1.55 insn per cycle - 2.796126198 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 371) (512y: 8) (512z: 0) +TOTAL : 2.767352 sec + 7,074,744,001 cycles:u # 2.552 GHz + 10,919,382,397 instructions:u # 1.54 insn per cycle + 2.775205710 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 412) (512y: 1) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_d_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_d_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_d_inl1/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_d_inl1/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 9.951438e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 9.951438e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 9.344111e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 9.344111e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 2.978139 sec - 7,064,137,452 cycles:u # 2.364 GHz - 10,423,152,716 instructions:u # 1.48 insn per cycle - 2.991902693 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 130) (512y: 4) (512z: 264) +TOTAL : 3.015544 sec + 7,170,726,069 cycles:u # 2.377 GHz + 10,478,969,947 instructions:u # 1.46 insn per cycle + 3.023095729 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 193) (512y: 2) (512z: 230) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_d_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_d_inl1/runTest.exe [ PASSED ] 3 tests. ========================================================================= diff --git a/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_f_inl0.txt b/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_f_inl0.txt index 1f0881574e..30e3d856fb 100644 --- a/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_f_inl0.txt +++ b/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_f_inl0.txt @@ -1,5 +1,5 @@ -/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum +/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum OMPFLAGS= AVX=none FPTYPE=f @@ -40,102 +40,102 @@ RNDGEN=curdev Building in BUILDDIR=build.512z_f_inl0 for tag=512z_f_inl0_curdev (USEBUILDDIR is set = 1) make: Nothing to be done for `all.512z_f_inl0_curdev'. -DATE: 2021-11-16_14:55:07 +DATE: 2021-12-01_10:08:12 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= -runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl0/gcheck.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl0/gcheck.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=2, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 1.567197e+09 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.249137e+09 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.567154e+09 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.236935e+09 ) sec^-1 MeanMatrixElemValue = ( 1.371686e-02 +- 3.270219e-06 ) GeV^0 -TOTAL : 0.427356 sec - 546,689,820 cycles:u # 1.296 GHz - 1,118,212,933 instructions:u # 2.05 insn per cycle - 0.481922205 seconds time elapsed +TOTAL : 0.842427 sec + 716,989,672 cycles:u # 0.704 GHz + 1,484,422,831 instructions:u # 2.07 insn per cycle + 1.126361376 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 56 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... ========================================================================= -runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl0/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl0/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=6, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.204350e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.204350e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.224589e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.224589e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371707e-02 +- 3.270376e-06 ) GeV^0 -TOTAL : 7.147557 sec - 18,988,594,465 cycles:u # 2.655 GHz - 47,311,641,086 instructions:u # 2.49 insn per cycle - 7.155123783 seconds time elapsed +TOTAL : 7.053663 sec + 18,764,093,689 cycles:u # 2.659 GHz + 47,311,642,026 instructions:u # 2.52 insn per cycle + 7.060851302 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 559) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl0/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_f_inl0/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_f_inl0/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=6, zero=0) Internal loops fptype_sv = VECTOR[4] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 4.412641e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.412641e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.573579e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.573579e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270375e-06 ) GeV^0 -TOTAL : 3.366959 sec - 8,896,429,813 cycles:u # 2.639 GHz - 19,646,258,706 instructions:u # 2.21 insn per cycle - 3.374957161 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 3997) (avx2: 0) (512y: 0) (512z: 0) +TOTAL : 3.344023 sec + 8,846,286,898 cycles:u # 2.642 GHz + 19,615,196,579 instructions:u # 2.22 insn per cycle + 3.351351467 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 3969) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_f_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_f_inl0/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_f_inl0/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_f_inl0/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=5, zero=0) Internal loops fptype_sv = VECTOR[8] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 8.181227e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 8.181227e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 8.181232e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 8.181232e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371705e-02 +- 3.270339e-06 ) GeV^0 -TOTAL : 2.676421 sec - 6,752,110,135 cycles:u # 2.519 GHz - 12,573,022,782 instructions:u # 1.86 insn per cycle - 2.684185694 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 3133) (512y: 0) (512z: 0) +TOTAL : 2.698787 sec + 6,823,046,265 cycles:u # 2.524 GHz + 12,524,650,277 instructions:u # 1.84 insn per cycle + 2.705979783 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 3175) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_f_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_f_inl0/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_f_inl0/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_f_inl0/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=5, zero=0) Internal loops fptype_sv = VECTOR[8] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 8.876739e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 8.876739e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 8.961058e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 8.961058e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371705e-02 +- 3.270339e-06 ) GeV^0 -TOTAL : 2.615215 sec - 6,618,025,602 cycles:u # 2.526 GHz - 12,525,838,377 instructions:u # 1.89 insn per cycle - 2.622760591 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 3023) (512y: 28) (512z: 0) +TOTAL : 2.587116 sec + 6,548,070,094 cycles:u # 2.527 GHz + 12,477,465,603 instructions:u # 1.91 insn per cycle + 2.594729483 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 3069) (512y: 26) (512z: 0) ------------------------------------------------------------------------- -runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_f_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_f_inl0/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_f_inl0/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_f_inl0/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=5, zero=0) Internal loops fptype_sv = VECTOR[16] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 7.315603e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 7.315603e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 7.493309e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 7.493309e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371705e-02 +- 3.270340e-06 ) GeV^0 -TOTAL : 2.756802 sec - 6,322,567,311 cycles:u # 2.291 GHz - 10,854,658,090 instructions:u # 1.72 insn per cycle - 2.764353327 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1674) (512y: 13) (512z: 2271) +TOTAL : 2.727509 sec + 6,281,035,658 cycles:u # 2.300 GHz + 10,798,421,531 instructions:u # 1.72 insn per cycle + 2.734747652 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1772) (512y: 12) (512z: 2256) ------------------------------------------------------------------------- -runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_f_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_f_inl0/runTest.exe [ PASSED ] 3 tests. ========================================================================= diff --git a/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_f_inl1.txt b/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_f_inl1.txt index af6dfdea32..adffd368b3 100644 --- a/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_f_inl1.txt +++ b/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_f_inl1.txt @@ -1,5 +1,5 @@ -/afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum +/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum OMPFLAGS= AVX=none FPTYPE=f @@ -40,102 +40,102 @@ RNDGEN=curdev Building in BUILDDIR=build.512z_f_inl1 for tag=512z_f_inl1_curdev (USEBUILDDIR is set = 1) make: Nothing to be done for `all.512z_f_inl1_curdev'. -DATE: 2021-10-30_18:39:46 +DATE: 2021-12-01_10:08:43 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl1/gcheck.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl1/gcheck.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=2, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 1.430147e+09 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.241660e+09 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.577348e+09 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.256680e+09 ) sec^-1 MeanMatrixElemValue = ( 1.371686e-02 +- 3.270219e-06 ) GeV^0 -TOTAL : 0.929019 sec - 731,390,576 cycles:u # 0.702 GHz - 1,497,204,981 instructions:u # 2.05 insn per cycle - 1.219085654 seconds time elapsed +TOTAL : 0.907275 sec + 718,084,580 cycles:u # 0.706 GHz + 1,482,928,433 instructions:u # 2.07 insn per cycle + 1.190328356 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 56 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl1/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl1/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=6, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.221707e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.221707e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.203281e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.203281e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270375e-06 ) GeV^0 -TOTAL : 2.440777 sec - 6,434,889,296 cycles:u # 2.631 GHz - 12,595,058,563 instructions:u # 1.96 insn per cycle - 2.448468922 seconds time elapsed +TOTAL : 2.431255 sec + 6,417,013,841 cycles:u # 2.634 GHz + 12,595,066,654 instructions:u # 1.96 insn per cycle + 2.438826820 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 796) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_f_inl1/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_f_inl1/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=6, zero=0) Internal loops fptype_sv = VECTOR[4] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 2.000845e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 2.000845e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.923450e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.923450e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270375e-06 ) GeV^0 -TOTAL : 2.254327 sec - 5,941,500,593 cycles:u # 2.630 GHz - 11,332,004,392 instructions:u # 1.91 insn per cycle - 2.262004870 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 503) (avx2: 0) (512y: 0) (512z: 0) +TOTAL : 2.297739 sec + 6,058,657,245 cycles:u # 2.632 GHz + 11,389,032,042 instructions:u # 1.88 insn per cycle + 2.304986309 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 508) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_f_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_f_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_f_inl1/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_f_inl1/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=5, zero=0) Internal loops fptype_sv = VECTOR[8] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 2.942985e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 2.942985e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 2.894889e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 2.894889e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371705e-02 +- 3.270339e-06 ) GeV^0 -TOTAL : 2.131788 sec - 5,510,571,346 cycles:u # 2.583 GHz - 9,702,504,319 instructions:u # 1.76 insn per cycle - 2.139640046 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 481) (512y: 0) (512z: 0) +TOTAL : 2.105405 sec + 5,454,622,456 cycles:u # 2.585 GHz + 9,655,713,280 instructions:u # 1.77 insn per cycle + 2.113229620 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 533) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_f_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_f_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_f_inl1/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_f_inl1/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=5, zero=0) Internal loops fptype_sv = VECTOR[8] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 3.054135e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.054135e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 3.028386e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.028386e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371705e-02 +- 3.270339e-06 ) GeV^0 -TOTAL : 2.105307 sec - 5,459,860,062 cycles:u # 2.588 GHz - 9,617,570,649 instructions:u # 1.76 insn per cycle - 2.112973584 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 440) (512y: 2) (512z: 0) +TOTAL : 2.093570 sec + 5,428,962,159 cycles:u # 2.588 GHz + 9,583,362,532 instructions:u # 1.77 insn per cycle + 2.101081895 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 497) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_f_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_f_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_f_inl1/check.exe -p 2048 256 12 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_f_inl1/check.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=5, zero=0) Internal loops fptype_sv = VECTOR[16] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 2.146895e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 2.146895e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 2.059238e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 2.059238e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371705e-02 +- 3.270340e-06 ) GeV^0 -TOTAL : 2.188251 sec - 5,431,722,356 cycles:u # 2.478 GHz - 9,341,535,791 instructions:u # 1.72 insn per cycle - 2.196084347 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 207) (512y: 0) (512z: 286) +TOTAL : 2.192812 sec + 5,434,991,260 cycles:u # 2.474 GHz + 9,332,494,253 instructions:u # 1.72 insn per cycle + 2.200097462 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 293) (512y: 0) (512z: 251) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_f_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_f_inl1/runTest.exe [ PASSED ] 3 tests. ========================================================================= From 32bf23b3b6388acdb425d4ca81349961093e5a51 Mon Sep 17 00:00:00 2001 From: Andrea Valassi Date: Wed, 1 Dec 2021 10:50:49 +0100 Subject: [PATCH 15/34] [klas5X] regenerate ggtt auto and run all tests - all ok, essentially the same performance --- .../gg_tt.auto/CODEGEN_cudacpp_gg_tt_log.txt | 28 +-- .../cudacpp/gg_tt.auto/SubProcesses/Memory.h | 19 +- .../P1_Sigma_sm_gg_ttx/CPPProcess.cc | 82 +++++--- .../P1_Sigma_sm_gg_ttx/CPPProcess.h | 14 +- .../P1_Sigma_sm_gg_ttx/check_sa.cc | 35 +--- .../gg_tt.auto/SubProcesses/runTest.cc | 36 ++-- .../gg_tt.auto/SubProcesses/testxxx.cc | 8 +- epochX/cudacpp/gg_tt.auto/src/HelAmps_sm.cc | 183 ++++++++++++++---- epochX/cudacpp/gg_tt.auto/src/HelAmps_sm.h | 20 +- epochX/cudacpp/gg_tt.auto/src/mgOnGpuConfig.h | 84 +++++--- .../cudacpp/gg_tt.auto/src/mgOnGpuVectors.h | 16 +- epochX/cudacpp/gg_tt.auto/src/rambo.cc | 6 +- epochX/cudacpp/gg_tt.auto/src/rambo.h | 6 +- .../logs_ggtt_auto/log_ggtt_auto_d_inl0.txt | 82 ++++---- .../logs_ggtt_auto/log_ggtt_auto_d_inl1.txt | 106 +++++----- .../logs_ggtt_auto/log_ggtt_auto_f_inl0.txt | 82 ++++---- .../logs_ggtt_auto/log_ggtt_auto_f_inl1.txt | 106 +++++----- 17 files changed, 537 insertions(+), 376 deletions(-) diff --git a/epochX/cudacpp/gg_tt.auto/CODEGEN_cudacpp_gg_tt_log.txt b/epochX/cudacpp/gg_tt.auto/CODEGEN_cudacpp_gg_tt_log.txt index 079df602a9..0d553f454d 100644 --- a/epochX/cudacpp/gg_tt.auto/CODEGEN_cudacpp_gg_tt_log.txt +++ b/epochX/cudacpp/gg_tt.auto/CODEGEN_cudacpp_gg_tt_log.txt @@ -50,7 +50,7 @@ generate g g > t t~ No model currently active, so we import the Standard Model INFO: load particles INFO: load vertices -DEBUG: model prefixing takes 0.006863117218017578  +DEBUG: model prefixing takes 0.006850719451904297  INFO: Restrict model sm with file models/sm/restrict_default.dat . INFO: Change particles name to pass to MG5 convention DEBUG: True [misc.py at line 2192]  @@ -82,20 +82,20 @@ INFO: Processing color information for process: g g > t t~ @1 DEBUG: type(subproc_group)= [output.py at line 139]  DEBUG: type(fortran_model)= [output.py at line 140]  INFO: Creating files in directory /data/avalassi/GPU2020/MG5aMC/2.7.0_gpu/CODEGEN_cudacpp_gg_tt/SubProcesses/P1_Sigma_sm_gg_ttx -DEBUG: Entering PLUGIN_OneProcessExporter.generate_process_files [model_handling.py at line 886]  +DEBUG: Entering PLUGIN_OneProcessExporter.generate_process_files [model_handling.py at line 893]  FileWriter for /data/avalassi/GPU2020/MG5aMC/2.7.0_gpu/CODEGEN_cudacpp_gg_tt/SubProcesses/P1_Sigma_sm_gg_ttx/./CPPProcess.h -DEBUG: Entering PLUGIN_OneProcessExporter.write_process_h_file [model_handling.py at line 935]  +DEBUG: Entering PLUGIN_OneProcessExporter.write_process_h_file [model_handling.py at line 942]  FileWriter for /data/avalassi/GPU2020/MG5aMC/2.7.0_gpu/CODEGEN_cudacpp_gg_tt/SubProcesses/P1_Sigma_sm_gg_ttx/./CPPProcess.cc -DEBUG: Entering PLUGIN_OneProcessExporter.write_process_cc_file [model_handling.py at line 957]  -DEBUG: call =  vxxxxx(allmomenta,m_pars->%s, cHel[ihel][%d],%+d, w_sv[%d], %d); [model_handling.py at line 1234]  -DEBUG: ('ZERO', 0, -1, 0, 0) [model_handling.py at line 1235]  -DEBUG: call =  vxxxxx(allmomenta,m_pars->%s, cHel[ihel][%d],%+d, w_sv[%d], %d); [model_handling.py at line 1234]  -DEBUG: ('ZERO', 1, -1, 1, 1) [model_handling.py at line 1235]  +DEBUG: Entering PLUGIN_OneProcessExporter.write_process_cc_file [model_handling.py at line 964]  +DEBUG: call =  vxxxxx(allmomenta,m_pars->%s, cHel[ihel][%d],%+d, w_sv[%d], %d); [model_handling.py at line 1241]  +DEBUG: ('ZERO', 0, -1, 0, 0) [model_handling.py at line 1242]  +DEBUG: call =  vxxxxx(allmomenta,m_pars->%s, cHel[ihel][%d],%+d, w_sv[%d], %d); [model_handling.py at line 1241]  +DEBUG: ('ZERO', 1, -1, 1, 1) [model_handling.py at line 1242]  DEBUG: only one Matrix-element supported?  INFO: Created files CPPProcess.h and CPPProcess.cc in directory /data/avalassi/GPU2020/MG5aMC/2.7.0_gpu/CODEGEN_cudacpp_gg_tt/SubProcesses/P1_Sigma_sm_gg_ttx/. -DEBUG: Entering PLUGIN_OneProcessExporter.edit_check_sa [model_handling.py at line 900]  -DEBUG: Entering PLUGIN_OneProcessExporter.edit_mgonGPU [model_handling.py at line 915]  -DEBUG: Entering PLUGIN_OneProcessExporter.edit_processidfile [model_handling.py at line 922]  +DEBUG: Entering PLUGIN_OneProcessExporter.edit_check_sa [model_handling.py at line 907]  +DEBUG: Entering PLUGIN_OneProcessExporter.edit_mgonGPU [model_handling.py at line 922]  +DEBUG: Entering PLUGIN_OneProcessExporter.edit_processidfile [model_handling.py at line 929]  Generated helas calls for 1 subprocesses (3 diagrams) in 0.008 s DEBUG: Entering PLUGIN_ProcessExporter.convert_model (create the model) [output.py at line 145]  ALOHA: aloha creates routines (starting by VVV1 with options P0) @@ -118,6 +118,6 @@ INFO: /data/avalassi/GPU2020/MG5aMC/2.7.0_gpu/CODEGEN_cudacpp_gg_tt/src/. and /d DEBUG: Entering PLUGIN_ProcessExporter.finalize [output.py at line 154]  quit -real 0m5.745s -user 0m0.771s -sys 0m0.129s +real 0m3.666s +user 0m0.772s +sys 0m0.133s diff --git a/epochX/cudacpp/gg_tt.auto/SubProcesses/Memory.h b/epochX/cudacpp/gg_tt.auto/SubProcesses/Memory.h index d7d1c7d4da..a9c08d3dd0 100644 --- a/epochX/cudacpp/gg_tt.auto/SubProcesses/Memory.h +++ b/epochX/cudacpp/gg_tt.auto/SubProcesses/Memory.h @@ -1,8 +1,9 @@ /* - * memory.h + * Memory.h * * Created on: 19.11.2020 * Author: shageboeck + * Modified: avalassi (vectors, alignment) */ #ifndef MEMORY_H @@ -46,13 +47,25 @@ std::unique_ptr> hstMakeUnique(std::size_t N) { #else +template +struct CppHstDeleter { + void operator()(T* mem) { + ::operator delete( mem, std::align_val_t{ mgOnGpu::cppAlign } ); + } +}; + template inline -std::unique_ptr hstMakeUnique(std::size_t N) { return std::unique_ptr{ new T[N]() }; }; +std::unique_ptr> hstMakeUnique(std::size_t N) { + // See https://www.bfilipek.com/2019/08/newnew-align.html#requesting-different-alignment + return std::unique_ptr>{ new( std::align_val_t{ mgOnGpu::cppAlign } ) T[N]() }; +}; #ifdef MGONGPU_CPPSIMD template<> inline -std::unique_ptr hstMakeUnique(std::size_t N) { return std::unique_ptr{ new fptype_v[N/neppV]() }; }; +std::unique_ptr> hstMakeUnique(std::size_t N) { + return std::unique_ptr>{ new( std::align_val_t{ mgOnGpu::cppAlign } ) fptype_v[N/neppV]() }; +}; #endif diff --git a/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/CPPProcess.cc b/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/CPPProcess.cc index 3a92e7549c..2d37aff5ea 100644 --- a/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/CPPProcess.cc +++ b/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/CPPProcess.cc @@ -71,10 +71,10 @@ namespace Proc __device__ INLINE void calculate_wavefunctions( int ihel, - const fptype_sv* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM - fptype_sv* allMEs // output: allMEs[npagM][neppM], final |M|^2 averaged over helicities + const fptype* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM + fptype* allMEs // output: allMEs[nevt], |M|^2 running_sum_over_helicities #ifndef __CUDACC__ - , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) #endif ) //ALWAYS_INLINE // attributes are not permitted in a function definition @@ -99,6 +99,9 @@ namespace Proc // === Calculate wavefunctions and amplitudes for all diagrams in all processes - Loop over nevt events === #ifndef __CUDACC__ const int npagV = nevt / neppV; +#ifdef MGONGPU_CPPSIMD + const bool isAligned_allMEs = ( (size_t)(allMEs) % mgOnGpu::cppAlign == 0 ); // require SIMD-friendly alignment by at least neppV*sizeof(fptype) +#endif // ** START LOOP ON IPAGV ** #ifdef _OPENMP // (NB gcc9 or higher, or clang, is required) @@ -106,7 +109,11 @@ namespace Proc // - shared: as the name says // - private: give each thread its own copy, without initialising // - firstprivate: give each thread its own copy, and initialise with value from outside +#ifdef MGONGPU_CPPSIMD +#pragma omp parallel for default(none) shared(allmomenta,allMEs,cHel,cIPC,cIPD,ihel,npagV,isAligned_allMEs) private (amp_sv,w_sv,jamp_sv) +#else #pragma omp parallel for default(none) shared(allmomenta,allMEs,cHel,cIPC,cIPD,ihel,npagV) private (amp_sv,w_sv,jamp_sv) +#endif #endif for ( int ipagV = 0; ipagV < npagV; ++ipagV ) #endif @@ -198,13 +205,27 @@ namespace Proc #ifdef __CUDACC__ const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid allMEs[ievt] += deltaMEs; - //printf( "calculate_wavefunction: %6d %2d %f\n", ievt, ihel, allMEs[ievt] ); + //if ( cNGoodHel > 0 ) printf( "calculate_wavefunction: %6d %2d %f\n", ievt, ihel, allMEs[ievt] ); +#else +#ifdef MGONGPU_CPPSIMD + if ( isAligned_allMEs ) + { + *reinterpret_cast( &( allMEs[ipagV*neppV] ) ) += deltaMEs; + } + else + { + for ( int ieppV=0; ieppV 0 ) + // for ( int ieppV=0; ieppV 0 ) printf( "calculate_wavefunction: %6d %2d %f\n", ipagV, ihel, allMEs[ipagV] ); +#endif #endif } - mgDebug( 1, __FUNCTION__ ); return; } @@ -248,10 +269,13 @@ namespace Proc memcpy( cHel, tHel, ncomb * mgOnGpu::npar * sizeof(short) ); #endif // SANITY CHECK: GPU memory usage may be based on casts of fptype[2] to cxtype - assert( sizeof(cxtype) == 2 * sizeof(fptype) ); + static_assert( sizeof(cxtype) == 2 * sizeof(fptype) ); #ifndef __CUDACC__ - // SANITY CHECK: momenta AOSOA uses vectors with the same size as fptype_v - assert( neppV == mgOnGpu::neppM ); + // SANITY CHECK: check that neppR, neppM and neppV are powers of two (https://stackoverflow.com/a/108360) + auto ispoweroftwo = []( int n ) { return ( n > 0 ) && !( n & ( n - 1 ) ); }; + static_assert( ispoweroftwo( mgOnGpu::neppR ) ); + static_assert( ispoweroftwo( mgOnGpu::neppM ) ); + static_assert( ispoweroftwo( neppV ) ); #endif } @@ -361,9 +385,9 @@ namespace Proc #ifdef __CUDACC__ __global__ - void sigmaKin_getGoodHel( const fptype_sv* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM - fptype_sv* allMEs, // output: allMEs[npagM][neppM], final |M|^2 averaged over helicities - bool* isGoodHel ) // output: isGoodHel[ncomb] - device array + void sigmaKin_getGoodHel( const fptype* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM + fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities + bool* isGoodHel ) // output: isGoodHel[ncomb] - device array { const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid // FIXME: assume process.nprocesses == 1 for the moment (eventually: need a loop over processes here?) @@ -381,33 +405,35 @@ namespace Proc } } #else - void sigmaKin_getGoodHel( const fptype_sv* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM - fptype_sv* allMEs, // output: allMEs[npagM][neppM], final |M|^2 averaged over helicities - bool* isGoodHel // output: isGoodHel[ncomb] - device array - , const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + void sigmaKin_getGoodHel( const fptype* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM + fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities + bool* isGoodHel // output: isGoodHel[ncomb] - device array + , const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) { + assert( (size_t)(allmomenta) % mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment + //assert( (size_t)(allMEs) % mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment const int maxtry0 = ( neppV > 16 ? neppV : 16 ); // 16, but at least neppV (otherwise the npagV loop does not even start) - fptype_sv allMEsLast[maxtry0/neppV] = { 0 }; + fptype allMEsLast[maxtry0] = { 0 }; const int maxtry = std::min( maxtry0, nevt ); // 16, but at most nevt (avoid invalid memory access if nevt allMEs[nevt*nprocesses]?) __global__ - void sigmaKin( const fptype_sv* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM - fptype_sv* allMEs // output: allMEs[npagM][neppM], final |M|^2 averaged over helicities + void sigmaKin( const fptype* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM + fptype* allMEs // output: allMEs[nevt], |M|^2 final_avg_over_helicities #ifndef __CUDACC__ - , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) #endif ) { @@ -477,7 +503,8 @@ namespace Proc const int npagV = nevt/neppV; for ( int ipagV = 0; ipagV < npagV; ++ipagV ) { - allMEs[ipagV] = fptype_sv{ 0 }; + for ( int ieppV=0; ieppV( nRnarray ); // AOSOA[npagR][nparf][np4][neppR] (NB: nevt=npagR*neppR) + auto hstRnarray = hstMakeUnique( nRnarray ); // AOSOA[npagR][nparf][np4][neppR] (NB: nevt=npagR*neppR) #endif - auto hstMomenta = hstMakeUnique( nMomenta ); // AOSOA[npagM][npar][np4][neppM] (NB: nevt=npagM*neppM) - auto hstIsGoodHel = hstMakeUnique( ncomb ); - auto hstWeights = hstMakeUnique( nWeights ); - auto hstMEs = hstMakeUnique( nMEs ); // AOSOA[npagM][neppM] (NB: nevt=npagM*neppM) + auto hstMomenta = hstMakeUnique( nMomenta ); // AOSOA[npagM][npar][np4][neppM] (NB: nevt=npagM*neppM) + auto hstIsGoodHel = hstMakeUnique( ncomb ); + auto hstWeights = hstMakeUnique( nWeights ); + auto hstMEs = hstMakeUnique( nMEs ); // ARRAY[nevt] #ifdef __CUDACC__ - auto devRnarray = devMakeUnique( nRnarray ); // AOSOA[npagR][nparf][np4][neppR] (NB: nevt=npagR*neppR) - auto devMomenta = devMakeUnique( nMomenta ); // AOSOA[npagM][npar][np4][neppM] (NB: nevt=npagM*neppM) - auto devIsGoodHel = devMakeUnique( ncomb ); - auto devWeights = devMakeUnique( nWeights ); - auto devMEs = devMakeUnique( nMEs ); // AOSOA[npagM][neppM] (NB: nevt=npagM*neppM) + auto devRnarray = devMakeUnique( nRnarray ); // AOSOA[npagR][nparf][np4][neppR] (NB: nevt=npagR*neppR) + auto devMomenta = devMakeUnique( nMomenta ); // AOSOA[npagM][npar][np4][neppM] (NB: nevt=npagM*neppM) + auto devIsGoodHel = devMakeUnique( ncomb ); + auto devWeights = devMakeUnique( nWeights ); + auto devMEs = devMakeUnique( nMEs ); // ARRAY[nevt] #if defined MGONGPU_CURAND_ONHOST or defined MGONGPU_COMMONRAND_ONHOST const int nbytesRnarray = nRnarray * sizeof(fptype); @@ -553,37 +553,22 @@ int main(int argc, char **argv) // NB: 'setw' affects only the next field (of any type) std::cout << std::scientific // fixed format: affects all floats (default precision: 6) << std::setw(4) << ipar + 1 -#ifndef MGONGPU_CPPSIMD << std::setw(14) << hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + 0*neppM + ieppM] // AOSOA[ipagM][ipar][0][ieppM] << std::setw(14) << hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + 1*neppM + ieppM] // AOSOA[ipagM][ipar][1][ieppM] << std::setw(14) << hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + 2*neppM + ieppM] // AOSOA[ipagM][ipar][2][ieppM] << std::setw(14) << hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + 3*neppM + ieppM] // AOSOA[ipagM][ipar][3][ieppM] -#else - << std::setw(14) << hstMomenta[ipagM*npar*np4 + ipar*np4 + 0][ieppM] // AOSOA[ipagM][ipar][0][ieppM] - << std::setw(14) << hstMomenta[ipagM*npar*np4 + ipar*np4 + 1][ieppM] // AOSOA[ipagM][ipar][1][ieppM] - << std::setw(14) << hstMomenta[ipagM*npar*np4 + ipar*np4 + 2][ieppM] // AOSOA[ipagM][ipar][2][ieppM] - << std::setw(14) << hstMomenta[ipagM*npar*np4 + ipar*np4 + 3][ieppM] // AOSOA[ipagM][ipar][3][ieppM] -#endif << std::endl << std::defaultfloat; // default format: affects all floats } std::cout << std::string(SEP79, '-') << std::endl; // Display matrix elements std::cout << " Matrix element = " -#ifndef MGONGPU_CPPSIMD << hstMEs[ievt] -#else - << hstMEs[ievt/neppM][ievt%neppM] -#endif << " GeV^" << meGeVexponent << std::endl; // FIXME: assume process.nprocesses == 1 std::cout << std::string(SEP79, '-') << std::endl; } // Fill the arrays with ALL MEs and weights -#ifndef MGONGPU_CPPSIMD matrixelementALL[iiter*nevt + ievt] = hstMEs[ievt]; // FIXME: assume process.nprocesses == 1 -#else - matrixelementALL[iiter*nevt + ievt] = hstMEs[ievt/neppM][ievt%neppM]; // FIXME: assume process.nprocesses == 1 -#endif weightALL[iiter*nevt + ievt] = hstWeights[ievt]; } diff --git a/epochX/cudacpp/gg_tt.auto/SubProcesses/runTest.cc b/epochX/cudacpp/gg_tt.auto/SubProcesses/runTest.cc index d92ffe4e7a..a4f3519434 100644 --- a/epochX/cudacpp/gg_tt.auto/SubProcesses/runTest.cc +++ b/epochX/cudacpp/gg_tt.auto/SubProcesses/runTest.cc @@ -21,7 +21,7 @@ template using unique_ptr_host = std::unique_ptr>; #else template -using unique_ptr_host = std::unique_ptr; +using unique_ptr_host = std::unique_ptr>; #endif struct CUDA_CPU_TestBase : public TestDriverBase { @@ -47,11 +47,11 @@ struct CPUTest : public CUDA_CPU_TestBase { // Struct data members (process, and memory structures for random numbers, momenta, matrix elements and weights on host and device) // [NB the hst/dev memory arrays must be initialised in the constructor, see issue #290] Proc::CPPProcess process; - unique_ptr_host hstRnarray; - unique_ptr_host hstMomenta; - unique_ptr_host hstIsGoodHel; - unique_ptr_host hstWeights; - unique_ptr_host hstMEs; + unique_ptr_host hstRnarray; + unique_ptr_host hstMomenta; + unique_ptr_host hstIsGoodHel; + unique_ptr_host hstWeights; + unique_ptr_host hstMEs; // Create a process object // Read param_card and set parameters @@ -61,11 +61,11 @@ struct CPUTest : public CUDA_CPU_TestBase { CPUTest( const std::string& refFileName ) : CUDA_CPU_TestBase( refFileName ), process(niter, gpublocks, gputhreads, /*verbose=*/false), - hstRnarray { hstMakeUnique( nRnarray ) }, // AOSOA[npagR][nparf][np4][neppR] - hstMomenta { hstMakeUnique( nMomenta ) }, // AOSOA[npagM][npar][np4][neppM] - hstIsGoodHel{ hstMakeUnique( mgOnGpu::ncomb ) }, - hstWeights { hstMakeUnique( nWeights ) }, - hstMEs { hstMakeUnique( nMEs ) } // AOSOA[npagM][neppM] + hstRnarray { hstMakeUnique( nRnarray ) }, // AOSOA[npagR][nparf][np4][neppR] + hstMomenta { hstMakeUnique( nMomenta ) }, // AOSOA[npagM][npar][np4][neppM] + hstIsGoodHel{ hstMakeUnique( mgOnGpu::ncomb ) }, + hstWeights { hstMakeUnique( nWeights ) }, + hstMEs { hstMakeUnique( nMEs ) } // ARRAY[nevt] { process.initProc("../../Cards/param_card.dat"); } @@ -106,19 +106,11 @@ struct CPUTest : public CUDA_CPU_TestBase { assert(particle < npar); const auto ipagM = evtNo / neppM; // #eventpage in this iteration const auto ieppM = evtNo % neppM; // #event in the current eventpage in this iteration -#ifndef MGONGPU_CPPSIMD return hstMomenta[ipagM*npar*np4*neppM + particle*np4*neppM + component*neppM + ieppM]; -#else - return hstMomenta[ipagM*npar*np4 + particle*np4 + component][ieppM]; -#endif }; fptype getMatrixElement(std::size_t ievt) const override { -#ifndef MGONGPU_CPPSIMD return hstMEs[ievt]; -#else - return hstMEs[ievt/neppV][ievt%neppV]; -#endif } }; @@ -161,12 +153,12 @@ struct CUDATest : public CUDA_CPU_TestBase { hstMomenta { hstMakeUnique( nMomenta ) }, // AOSOA[npagM][npar][np4][neppM] (nevt=npagM*neppM) hstIsGoodHel{ hstMakeUnique( mgOnGpu::ncomb ) }, hstWeights { hstMakeUnique( nWeights ) }, - hstMEs { hstMakeUnique( nMEs ) }, + hstMEs { hstMakeUnique( nMEs ) }, // ARRAY[nevt] devRnarray { devMakeUnique( nRnarray ) }, // AOSOA[npagR][nparf][np4][neppR] (nevt=npagR*neppR) - devMomenta { devMakeUnique( nMomenta ) }, + devMomenta { devMakeUnique( nMomenta ) }, // AOSOA[npagM][npar][np4][neppM] (nevt=npagM*neppM) devIsGoodHel{ devMakeUnique( mgOnGpu::ncomb ) }, devWeights { devMakeUnique( nWeights ) }, - devMEs { devMakeUnique( nMEs ) } + devMEs { devMakeUnique( nMEs ) } // ARRAY[nevt] { process.initProc("../../Cards/param_card.dat"); } diff --git a/epochX/cudacpp/gg_tt.auto/SubProcesses/testxxx.cc b/epochX/cudacpp/gg_tt.auto/SubProcesses/testxxx.cc index 6df896c75f..9ae06a7404 100644 --- a/epochX/cudacpp/gg_tt.auto/SubProcesses/testxxx.cc +++ b/epochX/cudacpp/gg_tt.auto/SubProcesses/testxxx.cc @@ -31,8 +31,8 @@ TEST( XTESTID_CPU( MG_EPOCH_PROCESS_ID ), testxxx ) assert( nevt % neppM == 0 ); // nevt must be a multiple of neppM // Fill in the input momenta const int nMomenta = np4 * npar * nevt; - auto hstMomenta = hstMakeUnique( nMomenta ); // AOSOA[npagM][npar=4][np4=4][neppM] - const fptype par0[np4 * nevt] = // AOS[nevt][np4] + auto hstMomenta = hstMakeUnique( nMomenta ); // AOSOA[npagM][npar=4][np4=4][neppM] + const fptype par0[np4 * nevt] = // AOS[nevt][np4] { 500, 0, 0, 500, // #0 (m=0 pT=0 E=pz>0) 500, 0, 0, -500, // #1 (m=0 pT=0 -E=pz<0) 500, 300, 400, 0, // #2 (m=0 pT>0 pz=0) @@ -72,11 +72,7 @@ TEST( XTESTID_CPU( MG_EPOCH_PROCESS_ID ), testxxx ) { const int ipagM = ievt/neppM; // #eventpage in this iteration const int ieppM = ievt%neppM; // #event in the current eventpage in this iteration -#ifdef MGONGPU_CPPSIMD - hstMomenta[ipagM*npar*np4 + ipar*np4 + ip4][ieppM] = par0[ievt*np4 + ip4]; // AOS to AOSOA -#else hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + ip4*neppM + ieppM] = par0[ievt*np4 + ip4]; // AOS to AOSOA -#endif } } // Expected output wavefunctions diff --git a/epochX/cudacpp/gg_tt.auto/src/HelAmps_sm.cc b/epochX/cudacpp/gg_tt.auto/src/HelAmps_sm.cc index 4abc728558..960fb91ffe 100644 --- a/epochX/cudacpp/gg_tt.auto/src/HelAmps_sm.cc +++ b/epochX/cudacpp/gg_tt.auto/src/HelAmps_sm.cc @@ -23,8 +23,8 @@ namespace MG5_sm //-------------------------------------------------------------------------- -#ifdef __CUDACC__ - // Return by reference + // Decode momentum AOSOA: compute address of fptype for the given particle, 4-momentum component and event + // Return the fptype by reference (equivalent to returning its memory address) __device__ inline const fptype& pIparIp4Ievt( const fptype* momenta, // input: momenta as AOSOA[npagM][npar][4][neppM] const int ipar, @@ -33,38 +33,149 @@ namespace MG5_sm { using mgOnGpu::np4; using mgOnGpu::npar; - const int neppM = mgOnGpu::neppM; // AOSOA layout: constant at compile-time - const int ipagM = ievt/neppM; // #eventpage in this iteration - const int ieppM = ievt%neppM; // #event in the current eventpage in this iteration - //printf( "%f\n", momenta[ipagM*npar*np4*neppM + ipar*np4*neppM + ip4*neppM + ieppM] ); + constexpr int neppM = mgOnGpu::neppM; // AOSOA layout: constant at compile-time + const int ipagM = ievt/neppM; // #event "M-page" + const int ieppM = ievt%neppM; // #event in the current event M-page + //printf( "%2d %2d %8d %8.3f\n", ipar, ip4, ievt, momenta[ipagM*npar*np4*neppM + ipar*np4*neppM + ip4*neppM + ieppM] ); return momenta[ipagM*npar*np4*neppM + ipar*np4*neppM + ip4*neppM + ieppM]; // AOSOA[ipagM][ipar][ip4][ieppM] //fptype (*momenta)[npar][np4][neppM] = (fptype (*)[npar][np4][neppM]) momenta; // cast to multiD array pointer (AOSOA) //return momenta[ipagM][ipar][ip4][ieppM]; // this seems ~1-2% faster in eemumu C++? } -#else - // Return by value: it seems a tiny bit faster than returning a reference (both for scalar and vector), not clear why + +#ifndef __CUDACC__ + // Return a SIMD vector of fptype's for neppV events (for the given particle, 4-momentum component and event "V-page") + // Return the vector by value: strictly speaking, this is only unavoidable for neppM=neppV (both being powers of 2), the momentum neppM-AOSOA is reinterpreted in terms of neppV-vectors: + // it could also be returned by reference, but no performance degradation is observed when returning by value inline - fptype_sv pIparIp4Ipag( const fptype_sv* momenta, // input: momenta as AOSOA[npagM][npar][4][neppM] + fptype_sv pIparIp4Ipag( const fptype* momenta, // input: momenta as AOSOA[npagM][npar][4][neppM] const int ipar, const int ip4, - const int ipagM ) + const int ipagV ) { - /* - //#ifndef MGONGPU_CPPSIMD - // TEMPORARY during epochX step3! Eventually remove this section (start) - // TEMPORARY during epochX step3! THERE IS NO SIMD YET: HENCE ipagM=ievt - // NB: this is needed for neppM>1 while neppV==1 (no SIMD) in eemumu.auto - // NB: this remains valid for neppM==1 with neppV==1 in eemumu (scalar) - return pIparIp4Ievt( momenta, ipar, ip4, ipagM ); - // TEMPORARY during epochX step3! Eventually remove this section (end) - //#else - */ - // NB: this assumes that neppV == neppM! - // NB: this is the same as "pIparIp4Ievt( momenta, ipar, ip4, ipagM )" for neppM==1 with neppV==1 - using mgOnGpu::np4; - using mgOnGpu::npar; - //printf( "%f\n", momenta[ipagM*npar*np4 + ipar*np4 + ip4] ); - return momenta[ipagM*npar*np4 + ipar*np4 + ip4]; // AOSOA[ipagM][ipar][ip4][ieppM] + const int ievt0 = ipagV*neppV; // virtual event V-page ipagV contains neppV events [ievt0...ievt0+neppV-1] +#ifdef MGONGPU_CPPSIMD + constexpr int neppM = mgOnGpu::neppM; // AOSOA layout: constant at compile-time + // Use c++17 "if constexpr": compile-time branching + if constexpr ( ( neppM >= neppV ) && ( neppM%neppV == 0 ) ) + { + constexpr bool useReinterpretCastIfPossible = true; // DEFAULT + //constexpr bool useReinterpretCastIfPossible = false; // FOR PERFORMANCE TESTS + constexpr bool skipAlignmentCheck = true; // DEFAULT (MAY SEGFAULT, NEEDS A SANITY CHECK ELSEWHERE!) + //constexpr bool skipAlignmentCheck = false; // SLOWER BUT SAFER + if constexpr ( useReinterpretCastIfPossible && skipAlignmentCheck ) + { + //static bool first=true; if( first ){ std::cout << "WARNING! skip alignment check" << std::endl; first=false; } // SLOWS DOWN... + // Fastest (4.92E6 in eemumu 512y) + // This requires alignment for momenta - segmentation fault otherwise! + return *reinterpret_cast( &( pIparIp4Ievt( momenta, ipar, ip4, ievt0 ) ) ); + } + else if ( useReinterpretCastIfPossible && ( (size_t)(momenta) % mgOnGpu::cppAlign == 0 ) ) + { + //static bool first=true; if( first ){ std::cout << "WARNING! alignment ok, use reinterpret cast" << std::endl; first=false; } // SLOWS DOWN... + // A bit (6%) slower (4.62E6 in eemumu 512y) because of the alignment check + // This explicitly checks alignment for momenta to avoid segmentation faults + return *reinterpret_cast( &( pIparIp4Ievt( momenta, ipar, ip4, ievt0 ) ) ); + } + else + { + //static bool first=true; if( first ){ std::cout << "WARNING! AOSOA but no reinterpret cast" << std::endl; first=false; } // SLOWS DOWN... + // A bit (2%) slower (4.86E6 in eemumu 512y) + // This does not require alignment for momenta, but it requires AOSOA with neppM>=neppV and neppM%neppV==0 + const fptype* out0 = &( pIparIp4Ievt( momenta, ipar, ip4, ievt0) ); +#if MGONGPU_CPPSIMD == 2 + return fptype_v{ *( out0 ), + *( out0+1 ) }; +#elif MGONGPU_CPPSIMD == 4 + return fptype_v{ *( out0 ), + *( out0+1 ), + *( out0+2 ), + *( out0+3 ) }; +#elif MGONGPU_CPPSIMD == 8 + return fptype_v{ *( out0 ), + *( out0+1 ), + *( out0+2 ), + *( out0+3 ), + *( out0+4 ), + *( out0+5 ), + *( out0+6 ), + *( out0+7 ) }; +#elif MGONGPU_CPPSIMD == 16 + return fptype_v{ *( out0 ), + *( out0+1 ), + *( out0+2 ), + *( out0+3 ), + *( out0+4 ), + *( out0+5 ), + *( out0+6 ), + *( out0+7 ), + *( out0+8 ), + *( out0+9 ), + *( out0+10 ), + *( out0+11 ), + *( out0+12 ), + *( out0+13 ), + *( out0+14 ), + *( out0+15 ) }; +#else +#warning Internal error? This code should never be reached! + // A bit (2%) slower (4.86E6 in eemumu 512y) + // This does not require alignment for momenta, but it requires AOSOA with neppM>=neppV and neppM%neppV==0 + fptype_v out; + for ( int ieppV=0; ieppV=neppV and neppM%neppV==0 (e.g. can be used with AOS neppM==1) +#if MGONGPU_CPPSIMD == 2 + return fptype_v{ pIparIp4Ievt( momenta, ipar, ip4, ievt0 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+1 ) }; +#elif MGONGPU_CPPSIMD == 4 + return fptype_v{ pIparIp4Ievt( momenta, ipar, ip4, ievt0 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+1 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+2 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+3 ) }; +#elif MGONGPU_CPPSIMD == 8 + return fptype_v{ pIparIp4Ievt( momenta, ipar, ip4, ievt0 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+1 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+2 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+3 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+4 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+5 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+6 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+7 ) }; +#elif MGONGPU_CPPSIMD == 16 + return fptype_v{ pIparIp4Ievt( momenta, ipar, ip4, ievt0 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+1 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+2 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+3 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+4 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+5 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+6 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+7 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+8 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+9 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+10 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+11 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+12 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+13 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+14 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+15 ) }; +#else +#warning Internal error? This code should never be reached! + // Much much (40%) slower (3.02E6 in eemumu 512y) + fptype_v out; + for ( int ieppV=0; ieppV 0) __device__ - void ipzxxx( const fptype_sv* momenta, + void ipzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -257,7 +368,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) __device__ - void imzxxx( const fptype_sv* momenta, + void imzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -306,7 +417,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PT > 0) __device__ - void ixzxxx( const fptype_sv* momenta, + void ixzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -367,7 +478,7 @@ namespace MG5_sm // Compute the output wavefunction vc[6] from the input momenta[npar*4*nevt] __device__ - void vxxxxx( const fptype_sv* momenta, + void vxxxxx( const fptype* momenta, const fptype vmass, // input: vector boson mass const int nhel, // input: -1, 0 (only if vmass!=0) or +1 (helicity of vector boson) const int nsv, // input: +1 (final) or -1 (initial) @@ -503,7 +614,7 @@ namespace MG5_sm // Compute the output wavefunction sc[3] from the input momenta[npar*4*nevt] __device__ - void sxxxxx( const fptype_sv* momenta, + void sxxxxx( const fptype* momenta, const fptype, // WARNING: input "smass" unused (missing in Fortran) - scalar boson mass const int, // WARNING: input "nhel" unused (missing in Fortran) - scalar has no helicity! const int nss, // input: +1 (final) or -1 (initial) @@ -544,7 +655,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] __device__ - void oxxxxx( const fptype_sv* momenta, + void oxxxxx( const fptype* momenta, const fptype fmass, // input: fermion mass const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -681,7 +792,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) __device__ - void opzxxx( const fptype_sv* momenta, + void opzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -730,7 +841,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) __device__ - void omzxxx( const fptype_sv* momenta, + void omzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -782,7 +893,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PT > 0) __device__ - void oxzxxx( const fptype_sv* momenta, + void oxzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) diff --git a/epochX/cudacpp/gg_tt.auto/src/HelAmps_sm.h b/epochX/cudacpp/gg_tt.auto/src/HelAmps_sm.h index ffdc0b2286..e1b6283da7 100644 --- a/epochX/cudacpp/gg_tt.auto/src/HelAmps_sm.h +++ b/epochX/cudacpp/gg_tt.auto/src/HelAmps_sm.h @@ -28,7 +28,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] __device__ INLINE - void ixxxxx( const fptype_sv* momenta, + void ixxxxx( const fptype* momenta, const fptype fmass, // input: fermion mass const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -43,7 +43,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) __device__ INLINE - void ipzxxx( const fptype_sv* momenta, + void ipzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -58,7 +58,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) __device__ INLINE - void imzxxx( const fptype_sv* momenta, + void imzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -73,7 +73,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PT > 0) __device__ INLINE - void ixzxxx( const fptype_sv* momenta, + void ixzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -87,7 +87,7 @@ namespace MG5_sm // Compute the output wavefunction vc[6] from the input momenta[npar*4*nevt] __device__ INLINE - void vxxxxx( const fptype_sv* momenta, + void vxxxxx( const fptype* momenta, const fptype vmass, // input: vector boson mass const int nhel, // input: -1, 0 (only if vmass!=0) or +1 (helicity of vector boson) const int nsv, // input: +1 (final) or -1 (initial) @@ -101,7 +101,7 @@ namespace MG5_sm // Compute the output wavefunction sc[3] from the input momenta[npar*4*nevt] __device__ INLINE - void sxxxxx( const fptype_sv* momenta, + void sxxxxx( const fptype* momenta, const fptype, // WARNING: input "smass" unused (missing in Fortran) - scalar boson mass const int, // WARNING: input "nhel" unused (missing in Fortran) - scalar has no helicity! const int nss, // input: +1 (final) or -1 (initial) @@ -115,7 +115,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] __device__ INLINE - void oxxxxx( const fptype_sv* momenta, + void oxxxxx( const fptype* momenta, const fptype fmass, // input: fermion mass const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -130,7 +130,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) __device__ INLINE - void opzxxx( const fptype_sv* momenta, + void opzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -145,7 +145,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) __device__ INLINE - void omzxxx( const fptype_sv* momenta, + void omzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -159,7 +159,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] __device__ INLINE - void oxzxxx( const fptype_sv* momenta, + void oxzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) diff --git a/epochX/cudacpp/gg_tt.auto/src/mgOnGpuConfig.h b/epochX/cudacpp/gg_tt.auto/src/mgOnGpuConfig.h index 39b12d3515..3b639644a3 100644 --- a/epochX/cudacpp/gg_tt.auto/src/mgOnGpuConfig.h +++ b/epochX/cudacpp/gg_tt.auto/src/mgOnGpuConfig.h @@ -96,47 +96,79 @@ namespace mgOnGpu //const int ntpbMAX = 256; // AV Apr2021: why had I set this to 256? const int ntpbMAX = 1024; // NB: 512 is ok, but 1024 does fail with "too many resources requested for launch" - // Vector sizes for AOSOA memory layouts (GPU coalesced memory access, CPU SIMD vectorization) - // (these are all best kept as a compile-time constants: see issue #23) + // Alignment requirement for using reinterpret_cast with SIMD vectorized code + // (using reinterpret_cast with non aligned memory may lead to segmentation faults!) +#ifndef __CUDACC__ + constexpr int cppAlign = 64; // alignment requirement for SIMD vectorization (64-byte i.e. 512-bit) +#endif - // Number of Events Per Page in the momenta AOSOA memory layout + // C++ SIMD vectorization width (this will be used to set neppV) #ifdef __CUDACC__ #undef MGONGPU_CPPSIMD - // ----------------------------------------------------------------------------------------- - // --- GPUs: neppM must be a power of 2 times the number of fptype's in a 32-byte cacheline - // --- This is relevant to ensure coalesced access to momenta in global memory - // --- Note that neppR is hardcoded and may differ from neppM and neppV on some platforms - // ----------------------------------------------------------------------------------------- - //const int neppM = 64/sizeof(fptype); // 2x 32-byte GPU cache lines: 8 (DOUBLE) or 16 (FLOAT) - const int neppM = 32/sizeof(fptype); // (DEFAULT) 32-byte GPU cache line: 4 (DOUBLE) or 8 (FLOAT) - //const int neppM = 1; // *** NB: this is equivalent to AOS *** - //const int neppM = 32; // older default #else - // ----------------------------------------------------------------------------------------- - // --- CPUs: neppM must be exactly equal to the number of fptype's in a vector register - // --- [DEFAULT is 256-width AVX512 aka "512y" on CPUs, 32-byte as GPUs, faster than AVX2] - // --- The logic of the code requires the size neppV of fptype_v to be equal to neppM - // --- Note that neppR is hardcoded and may differ from neppM and neppV on some platforms - // ----------------------------------------------------------------------------------------- #if defined __AVX512VL__ -#define MGONGPU_CPPSIMD 1 #ifdef MGONGPU_PVW512 - const int neppM = 64/sizeof(fptype); // "512z" AVX512 with 512 width (512-bit ie 64-byte): 8 (DOUBLE) or 16 (FLOAT) + // "512z" AVX512 with 512 width (512-bit ie 64-byte): 8 (DOUBLE) or 16 (FLOAT) +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 8 #else - const int neppM = 32/sizeof(fptype); // "512y" AVX512 with 256 width (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [gcc DEFAULT] +#define MGONGPU_CPPSIMD 16 +#endif +#else + // "512y" AVX512 with 256 width (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [gcc DEFAULT] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 4 +#else +#define MGONGPU_CPPSIMD 8 +#endif #endif #elif defined __AVX2__ -#define MGONGPU_CPPSIMD 1 - const int neppM = 32/sizeof(fptype); // "avx2" AVX2 (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [clang DEFAULT] + // "avx2" AVX2 (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [clang DEFAULT] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 4 +#else +#define MGONGPU_CPPSIMD 8 +#endif #elif defined __SSE4_2__ -#define MGONGPU_CPPSIMD 1 - const int neppM = 16/sizeof(fptype); // "sse4" SSE4.2 (128-bit ie 16-byte): 2 (DOUBLE) or 4 (FLOAT) + // "sse4" SSE4.2 (128-bit ie 16-byte): 2 (DOUBLE) or 4 (FLOAT) +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 2 #else +#define MGONGPU_CPPSIMD 4 +#endif +#else + // "none" i.e. no SIMD (*** NB: this is equivalent to AOS ***) #undef MGONGPU_CPPSIMD - const int neppM = 1; // "none" i.e. no SIMD (*** NB: this is equivalent to AOS ***) #endif #endif + // Number of Events Per Page in the momenta AOSOA memory layout +#ifdef __CUDACC__ + // ----------------------------------------------------------------------------------------------- + // --- GPUs: neppM is best set to a power of 2 times the number of fptype's in a 32-byte cacheline + // --- This is relevant to ensure coalesced access to momenta in global memory + // --- Note that neppR is hardcoded and may differ from neppM and neppV on some platforms + // ----------------------------------------------------------------------------------------------- + //const int neppM = 64/sizeof(fptype); // 2x 32-byte GPU cache lines (512 bits): 8 (DOUBLE) or 16 (FLOAT) + const int neppM = 32/sizeof(fptype); // (DEFAULT) 32-byte GPU cache line (256 bits): 4 (DOUBLE) or 8 (FLOAT) + //const int neppM = 1; // *** NB: this is equivalent to AOS *** +#else + // ----------------------------------------------------------------------------------------------- + // --- CPUs: neppM is best set equal to the number of fptype's (neppV) in a vector register + // --- This is relevant to ensure faster access to momenta from C++ memory cache lines + // --- However, neppM is now decoupled from neppV (issue #176) and can be separately hardcoded + // --- In practice, neppR, neppM and neppV could now (in principle) all be different + // ----------------------------------------------------------------------------------------------- +#ifdef MGONGPU_CPPSIMD + const int neppM = MGONGPU_CPPSIMD; // (DEFAULT) neppM=neppV for optimal performance +#else + const int neppM = 1; // (DEFAULT) neppM=neppV for optimal performance (NB: this is equivalent to AOS) +#endif + //const int neppM = 64/sizeof(fptype); // maximum CPU vector width (512 bits): 8 (DOUBLE) or 16 (FLOAT) + //const int neppM = 32/sizeof(fptype); // lower CPU vector width (256 bits): 4 (DOUBLE) or 8 (FLOAT) + //const int neppM = 1; // *** NB: this is equivalent to AOS *** +#endif + // Number of Events Per Page in the random number AOSOA memory layout // *** NB Different values of neppR lead to different physics results: the *** // *** same 1d array is generated, but it is interpreted in different ways *** diff --git a/epochX/cudacpp/gg_tt.auto/src/mgOnGpuVectors.h b/epochX/cudacpp/gg_tt.auto/src/mgOnGpuVectors.h index e1da365663..4e975b2ca1 100644 --- a/epochX/cudacpp/gg_tt.auto/src/mgOnGpuVectors.h +++ b/epochX/cudacpp/gg_tt.auto/src/mgOnGpuVectors.h @@ -13,9 +13,13 @@ namespace mgOnGpu { + #ifdef MGONGPU_CPPSIMD - const int neppV = neppM; + const int neppV = MGONGPU_CPPSIMD; + + // SANITY CHECK: cppAlign must be a multiple of neppV * sizeof(fptype) + static_assert( mgOnGpu::cppAlign % ( neppV * sizeof(fptype) ) == 0 ); // --- Type definition (using vector compiler extensions: need -march=...) #ifdef __clang__ // https://clang.llvm.org/docs/LanguageExtensions.html#vectors-and-extended-vectors @@ -96,7 +100,7 @@ namespace mgOnGpu #else // MGONGPU_CPPSIMD not defined - const int neppV = 1; // Note: also neppM is equal to 1 + const int neppV = 1; #endif } @@ -114,7 +118,7 @@ using mgOnGpu::bool_v; inline std::ostream& operator<<( std::ostream& out, const bool_v& v ) { out << "{ " << v[0]; - for ( int i=1; i #include @@ -22,7 +22,7 @@ namespace rambo2toNm0 // [NB: the output buffer includes both initial and final momenta, but only initial momenta are filled in] __global__ void getMomentaInitial( const fptype energy, // input: energy - fptype_sv momenta1d[] // output: momenta as AOSOA[npagM][npar][4][neppM] + fptype momenta1d[] // output: momenta as AOSOA[npagM][npar][4][neppM] #ifndef __CUDACC__ , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) #endif @@ -63,7 +63,7 @@ namespace rambo2toNm0 __global__ void getMomentaFinal( const fptype energy, // input: energy const fptype rnarray1d[], // input: random numbers in [0,1] as AOSOA[npagR][nparf][4][neppR] - fptype_sv momenta1d[], // output: momenta as AOSOA[npagM][npar][4][neppM] + fptype momenta1d[], // output: momenta as AOSOA[npagM][npar][4][neppM] fptype wgts[] // output: weights[nevt] #ifndef __CUDACC__ , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) diff --git a/epochX/cudacpp/gg_tt.auto/src/rambo.h b/epochX/cudacpp/gg_tt.auto/src/rambo.h index 030526759f..9d9e876fe3 100644 --- a/epochX/cudacpp/gg_tt.auto/src/rambo.h +++ b/epochX/cudacpp/gg_tt.auto/src/rambo.h @@ -3,7 +3,7 @@ #include "mgOnGpuConfig.h" #include "mgOnGpuTypes.h" -#include "mgOnGpuVectors.h" +//#include "mgOnGpuVectors.h" #include @@ -47,7 +47,7 @@ namespace rambo2toNm0 // [NB: the output buffer includes both initial and final momenta, but only initial momenta are filled in] __global__ void getMomentaInitial( const fptype energy, // input: energy - fptype_sv momenta1d[] // output: momenta as AOSOA[npagM][npar][4][neppM] + fptype momenta1d[] // output: momenta as AOSOA[npagM][npar][4][neppM] #ifndef __CUDACC__ , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) #endif @@ -60,7 +60,7 @@ namespace rambo2toNm0 __global__ void getMomentaFinal( const fptype energy, // input: energy const fptype rnarray1d[], // input: random numbers in [0,1] as AOSOA[npagR][nparf][4][neppR] - fptype_sv momenta1d[], // output: momenta as AOSOA[npagM][npar][4][neppM] + fptype momenta1d[], // output: momenta as AOSOA[npagM][npar][4][neppM] fptype wgts[] // output: weights[nevt] #ifndef __CUDACC__ , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) diff --git a/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_d_inl0.txt b/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_d_inl0.txt index 99201a5a2a..f223ac29b8 100644 --- a/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_d_inl0.txt +++ b/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_d_inl0.txt @@ -40,20 +40,20 @@ RNDGEN=curdev Building in BUILDDIR=build.512z_d_inl0 for tag=512z_d_inl0_curdev (USEBUILDDIR is set = 1) make: Nothing to be done for `all.512z_d_inl0_curdev'. -DATE: 2021-11-16_14:27:59 +DATE: 2021-12-01_10:48:37 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.none_d_inl0/gcheck.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTX_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 1.312981e+08 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.431548e+08 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.312795e+08 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.432637e+08 ) sec^-1 MeanMatrixElemValue = ( 2.085623e+00 +- 4.835084e-03 ) GeV^0 -TOTAL : 0.479925 sec - 132,551,339 cycles:u # 0.554 GHz - 166,405,911 instructions:u # 1.26 insn per cycle - 0.538401897 seconds time elapsed +TOTAL : 1.113676 sec + 1,224,992,382 cycles:u # 1.002 GHz + 2,374,241,054 instructions:u # 1.94 insn per cycle + 1.402465765 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 172 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... @@ -62,13 +62,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.979357e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.979357e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 2.010504e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 2.010504e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085623e+00 +- 4.835084e-03 ) GeV^0 -TOTAL : 2.878387 sec - 7,599,237,579 cycles:u # 2.637 GHz - 22,151,432,604 instructions:u # 2.91 insn per cycle - 2.885839402 seconds time elapsed +TOTAL : 2.835012 sec + 7,496,138,252 cycles:u # 2.641 GHz + 22,151,432,219 instructions:u # 2.96 insn per cycle + 2.842413425 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 552) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.none_d_inl0/runTest.exe @@ -78,14 +78,14 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[2] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 3.125751e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.125751e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 3.152961e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.152961e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085623e+00 +- 4.835084e-03 ) GeV^0 -TOTAL : 1.906885 sec - 5,010,161,810 cycles:u # 2.621 GHz - 13,225,752,134 instructions:u # 2.64 insn per cycle - 1.914228036 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 2632) (avx2: 0) (512y: 0) (512z: 0) +TOTAL : 1.892329 sec + 4,974,774,276 cycles:u # 2.622 GHz + 13,213,234,724 instructions:u # 2.66 insn per cycle + 1.900209238 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 2602) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.sse4_d_inl0/runTest.exe [ PASSED ] 3 tests. @@ -94,14 +94,14 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 5.667374e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 5.667374e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 5.578043e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 5.578043e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085623e+00 +- 4.835084e-03 ) GeV^0 -TOTAL : 1.147653 sec - 2,615,098,820 cycles:u # 2.269 GHz - 5,230,318,597 instructions:u # 2.00 insn per cycle - 1.155314354 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2274) (512y: 0) (512z: 0) +TOTAL : 1.163111 sec + 2,648,976,099 cycles:u # 2.269 GHz + 5,240,076,824 instructions:u # 1.98 insn per cycle + 1.170428063 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2299) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.avx2_d_inl0/runTest.exe [ PASSED ] 3 tests. @@ -110,14 +110,14 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 6.241195e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 6.241195e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 6.311632e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 6.311632e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085623e+00 +- 4.835084e-03 ) GeV^0 -TOTAL : 1.065271 sec - 2,429,507,480 cycles:u # 2.272 GHz - 5,015,879,379 instructions:u # 2.06 insn per cycle - 1.072717730 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2145) (512y: 59) (512z: 0) +TOTAL : 1.053746 sec + 2,403,014,525 cycles:u # 2.270 GHz + 5,029,831,866 instructions:u # 2.09 insn per cycle + 1.061475986 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2170) (512y: 52) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512y_d_inl0/runTest.exe [ PASSED ] 3 tests. @@ -126,14 +126,14 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 3.822751e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.822751e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 3.778810e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.778810e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085623e+00 +- 4.835084e-03 ) GeV^0 -TOTAL : 1.597020 sec - 2,683,864,834 cycles:u # 1.676 GHz - 3,389,616,735 instructions:u # 1.26 insn per cycle - 1.604518166 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1190) (512y: 68) (512z: 1694) +TOTAL : 1.613268 sec + 2,705,674,570 cycles:u # 1.673 GHz + 3,394,918,575 instructions:u # 1.25 insn per cycle + 1.620793314 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1258) (512y: 66) (512z: 1678) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512z_d_inl0/runTest.exe [ PASSED ] 3 tests. diff --git a/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_d_inl1.txt b/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_d_inl1.txt index d491121e99..3430f3ea77 100644 --- a/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_d_inl1.txt +++ b/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_d_inl1.txt @@ -1,5 +1,5 @@ -/afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx +/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx OMPFLAGS= AVX=none FPTYPE=d @@ -40,102 +40,102 @@ RNDGEN=curdev Building in BUILDDIR=build.512z_d_inl1 for tag=512z_d_inl1_curdev (USEBUILDDIR is set = 1) make: Nothing to be done for `all.512z_d_inl1_curdev'. -DATE: 2021-10-30_18:36:40 +DATE: 2021-12-01_10:48:59 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.none_d_inl1/gcheck.exe -p 2048 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.none_d_inl1/gcheck.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTX_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 1.304274e+08 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.422581e+08 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.306291e+08 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.423126e+08 ) sec^-1 MeanMatrixElemValue = ( 2.085623e+00 +- 4.835084e-03 ) GeV^0 -TOTAL : 1.059312 sec - 1,244,172,993 cycles:u # 1.005 GHz - 2,376,257,343 instructions:u # 1.91 insn per cycle - 1.353917565 seconds time elapsed +TOTAL : 1.024492 sec + 1,222,382,669 cycles:u # 1.010 GHz + 2,372,751,454 instructions:u # 1.94 insn per cycle + 1.312533047 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 172 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.none_d_inl1/check.exe -p 2048 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.none_d_inl1/check.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 2.327101e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 2.327101e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 2.341609e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 2.341609e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085623e+00 +- 4.835084e-03 ) GeV^0 -TOTAL : 2.481966 sec - 6,548,306,244 cycles:u # 2.633 GHz - 16,136,611,937 instructions:u # 2.46 insn per cycle - 2.489735462 seconds time elapsed +TOTAL : 2.465367 sec + 6,506,955,421 cycles:u # 2.635 GHz + 16,136,621,169 instructions:u # 2.48 insn per cycle + 2.472952243 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 676) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.none_d_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.none_d_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.sse4_d_inl1/check.exe -p 2048 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.sse4_d_inl1/check.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[2] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 3.099730e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.099730e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 3.112549e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.112549e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085623e+00 +- 4.835084e-03 ) GeV^0 -TOTAL : 1.922102 sec - 5,049,614,164 cycles:u # 2.621 GHz - 10,269,554,660 instructions:u # 2.03 insn per cycle - 1.929961367 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 2457) (avx2: 0) (512y: 0) (512z: 0) +TOTAL : 1.914624 sec + 5,030,295,164 cycles:u # 2.621 GHz + 10,269,628,611 instructions:u # 2.04 insn per cycle + 1.921809425 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 2413) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.sse4_d_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.sse4_d_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.avx2_d_inl1/check.exe -p 2048 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.avx2_d_inl1/check.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 4.651374e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.651374e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.678398e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.678398e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085623e+00 +- 4.835084e-03 ) GeV^0 -TOTAL : 1.353633 sec - 3,070,111,385 cycles:u # 2.263 GHz - 5,102,973,077 instructions:u # 1.66 insn per cycle - 1.361338729 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2566) (512y: 0) (512z: 0) +TOTAL : 1.341163 sec + 3,052,359,057 cycles:u # 2.271 GHz + 5,123,224,339 instructions:u # 1.68 insn per cycle + 1.348125127 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2607) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.avx2_d_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.avx2_d_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512y_d_inl1/check.exe -p 2048 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512y_d_inl1/check.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 4.662222e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.662222e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.759932e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.759932e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085623e+00 +- 4.835084e-03 ) GeV^0 -TOTAL : 1.349824 sec - 3,068,441,775 cycles:u # 2.266 GHz - 4,703,978,622 instructions:u # 1.53 insn per cycle - 1.357451626 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2173) (512y: 194) (512z: 0) +TOTAL : 1.324804 sec + 3,018,474,640 cycles:u # 2.271 GHz + 4,738,911,570 instructions:u # 1.57 insn per cycle + 1.332081839 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2215) (512y: 187) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512y_d_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512y_d_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512z_d_inl1/check.exe -p 2048 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512z_d_inl1/check.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 3.542404e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.542404e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 3.516536e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.516536e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085623e+00 +- 4.835084e-03 ) GeV^0 -TOTAL : 1.711320 sec - 2,849,638,832 cycles:u # 1.663 GHz - 3,728,147,616 instructions:u # 1.31 insn per cycle - 1.719457974 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1486) (512y: 256) (512z: 1660) +TOTAL : 1.717694 sec + 2,872,588,243 cycles:u # 1.668 GHz + 3,812,896,988 instructions:u # 1.33 insn per cycle + 1.725199428 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1529) (512y: 256) (512z: 1635) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512z_d_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512z_d_inl1/runTest.exe [ PASSED ] 3 tests. ========================================================================= diff --git a/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_f_inl0.txt b/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_f_inl0.txt index 7531acce80..7182b6503d 100644 --- a/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_f_inl0.txt +++ b/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_f_inl0.txt @@ -40,20 +40,20 @@ RNDGEN=curdev Building in BUILDDIR=build.512z_f_inl0 for tag=512z_f_inl0_curdev (USEBUILDDIR is set = 1) make: Nothing to be done for `all.512z_f_inl0_curdev'. -DATE: 2021-11-16_14:49:08 +DATE: 2021-12-01_10:49:20 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.none_f_inl0/gcheck.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTX_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 3.404423e+08 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.839363e+08 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 3.383804e+08 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.808356e+08 ) sec^-1 MeanMatrixElemValue = ( 2.085652e+00 +- 4.835323e-03 ) GeV^0 -TOTAL : 0.424261 sec - 122,494,952 cycles:u # 0.568 GHz - 153,432,560 instructions:u # 1.25 insn per cycle - 0.476778459 seconds time elapsed +TOTAL : 0.852993 sec + 741,956,461 cycles:u # 0.736 GHz + 1,483,788,078 instructions:u # 2.00 insn per cycle + 1.143851305 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 96 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... @@ -62,13 +62,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.896826e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.896826e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.916540e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.916540e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085710e+00 +- 4.835783e-03 ) GeV^0 -TOTAL : 2.945622 sec - 7,810,880,840 cycles:u # 2.648 GHz - 22,160,208,246 instructions:u # 2.84 insn per cycle - 2.953233508 seconds time elapsed +TOTAL : 2.916806 sec + 7,746,567,982 cycles:u # 2.652 GHz + 22,160,206,610 instructions:u # 2.86 insn per cycle + 2.924231787 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 500) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.none_f_inl0/runTest.exe @@ -78,14 +78,14 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 4.571217e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.571217e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.577346e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.577346e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085711e+00 +- 4.835793e-03 ) GeV^0 -TOTAL : 1.329725 sec - 3,499,186,459 cycles:u # 2.623 GHz - 8,211,118,393 instructions:u # 2.35 insn per cycle - 1.337343096 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 3287) (avx2: 0) (512y: 0) (512z: 0) +TOTAL : 1.334202 sec + 3,504,817,250 cycles:u # 2.624 GHz + 8,219,015,975 instructions:u # 2.35 insn per cycle + 1.342614165 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 3259) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.sse4_f_inl0/runTest.exe [ PASSED ] 3 tests. @@ -94,14 +94,14 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.064710e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.064710e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.044839e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.044839e+06 ) sec^-1 MeanMatrixElemValue = ( 2.085720e+00 +- 4.835915e-03 ) GeV^0 -TOTAL : 0.673390 sec - 1,546,623,916 cycles:u # 2.283 GHz - 3,160,096,079 instructions:u # 2.04 insn per cycle - 0.681156420 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2587) (512y: 0) (512z: 0) +TOTAL : 0.679389 sec + 1,567,021,135 cycles:u # 2.292 GHz + 3,160,253,157 instructions:u # 2.02 insn per cycle + 0.686556102 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2629) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.avx2_f_inl0/runTest.exe [ PASSED ] 3 tests. @@ -110,14 +110,14 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.143307e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.143307e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.137933e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.137933e+06 ) sec^-1 MeanMatrixElemValue = ( 2.085720e+00 +- 4.835915e-03 ) GeV^0 -TOTAL : 0.637127 sec - 1,471,068,759 cycles:u # 2.293 GHz - 3,048,420,178 instructions:u # 2.07 insn per cycle - 0.644522877 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2455) (512y: 28) (512z: 0) +TOTAL : 0.638720 sec + 1,471,997,982 cycles:u # 2.288 GHz + 3,050,674,328 instructions:u # 2.07 insn per cycle + 0.646338395 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2501) (512y: 26) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512y_f_inl0/runTest.exe [ PASSED ] 3 tests. @@ -126,14 +126,14 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[16] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 8.204639e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 8.204639e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 8.184860e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 8.184860e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085721e+00 +- 4.835915e-03 ) GeV^0 -TOTAL : 0.817551 sec - 1,434,342,802 cycles:u # 1.745 GHz - 2,178,928,211 instructions:u # 1.52 insn per cycle - 0.825410303 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1608) (512y: 13) (512z: 1794) +TOTAL : 0.818624 sec + 1,432,989,702 cycles:u # 1.742 GHz + 2,177,381,973 instructions:u # 1.52 insn per cycle + 0.825819338 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1706) (512y: 12) (512z: 1779) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512z_f_inl0/runTest.exe [ PASSED ] 3 tests. diff --git a/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_f_inl1.txt b/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_f_inl1.txt index a75032b635..345b301771 100644 --- a/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_f_inl1.txt +++ b/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_f_inl1.txt @@ -1,5 +1,5 @@ -/afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx +/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx OMPFLAGS= AVX=none FPTYPE=f @@ -40,102 +40,102 @@ RNDGEN=curdev Building in BUILDDIR=build.512z_f_inl1 for tag=512z_f_inl1_curdev (USEBUILDDIR is set = 1) make: Nothing to be done for `all.512z_f_inl1_curdev'. -DATE: 2021-10-30_18:37:24 +DATE: 2021-12-01_10:49:40 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.none_f_inl1/gcheck.exe -p 2048 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.none_f_inl1/gcheck.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTX_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 3.374757e+08 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.826836e+08 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 3.378138e+08 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.803138e+08 ) sec^-1 MeanMatrixElemValue = ( 2.085652e+00 +- 4.835323e-03 ) GeV^0 -TOTAL : 0.827076 sec - 744,170,702 cycles:u # 0.736 GHz - 1,485,269,139 instructions:u # 2.00 insn per cycle - 1.113117285 seconds time elapsed +TOTAL : 0.831964 sec + 739,027,153 cycles:u # 0.740 GHz + 1,483,617,609 instructions:u # 2.01 insn per cycle + 1.113943483 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 96 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.none_f_inl1/check.exe -p 2048 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.none_f_inl1/check.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 2.466016e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 2.466016e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 2.492472e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 2.492472e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085710e+00 +- 4.835783e-03 ) GeV^0 -TOTAL : 2.308595 sec - 6,115,836,092 cycles:u # 2.643 GHz - 16,170,552,652 instructions:u # 2.64 insn per cycle - 2.316582613 seconds time elapsed +TOTAL : 2.284811 sec + 6,054,859,307 cycles:u # 2.646 GHz + 16,170,560,643 instructions:u # 2.67 insn per cycle + 2.292112704 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 682) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.none_f_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.none_f_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.sse4_f_inl1/check.exe -p 2048 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.sse4_f_inl1/check.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 5.757513e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 5.757513e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 5.691714e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 5.691714e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085711e+00 +- 4.835793e-03 ) GeV^0 -TOTAL : 1.094665 sec - 2,872,284,293 cycles:u # 2.611 GHz - 6,482,358,161 instructions:u # 2.26 insn per cycle - 1.102662026 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 2766) (avx2: 0) (512y: 0) (512z: 0) +TOTAL : 1.104665 sec + 2,903,922,806 cycles:u # 2.618 GHz + 6,525,917,384 instructions:u # 2.25 insn per cycle + 1.111912784 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 2785) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.sse4_f_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.sse4_f_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.avx2_f_inl1/check.exe -p 2048 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.avx2_f_inl1/check.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 7.286173e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 7.286173e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 7.402418e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 7.402418e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085720e+00 +- 4.835915e-03 ) GeV^0 -TOTAL : 0.899104 sec - 2,060,553,995 cycles:u # 2.281 GHz - 3,658,783,634 instructions:u # 1.78 insn per cycle - 0.906808018 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 3511) (512y: 0) (512z: 0) +TOTAL : 0.883470 sec + 2,031,553,377 cycles:u # 2.289 GHz + 3,704,033,940 instructions:u # 1.82 insn per cycle + 0.890330929 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 3503) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.avx2_f_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.avx2_f_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512y_f_inl1/check.exe -p 2048 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512y_f_inl1/check.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 7.347475e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 7.347475e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 7.212886e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 7.212886e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085720e+00 +- 4.835915e-03 ) GeV^0 -TOTAL : 0.893926 sec - 2,051,569,062 cycles:u # 2.283 GHz - 3,495,726,024 instructions:u # 1.70 insn per cycle - 0.901717222 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 3327) (512y: 26) (512z: 0) +TOTAL : 0.903901 sec + 2,076,333,677 cycles:u # 2.286 GHz + 3,578,727,346 instructions:u # 1.72 insn per cycle + 0.910968670 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 3338) (512y: 24) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512y_f_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512y_f_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512z_f_inl1/check.exe -p 2048 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512z_f_inl1/check.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[16] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 5.749691e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 5.749691e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 5.695668e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 5.695668e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085721e+00 +- 4.835915e-03 ) GeV^0 -TOTAL : 1.092575 sec - 1,862,504,609 cycles:u # 1.696 GHz - 3,016,783,055 instructions:u # 1.62 insn per cycle - 1.102388581 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 3291) (512y: 0) (512z: 1973) +TOTAL : 1.097831 sec + 1,872,089,939 cycles:u # 1.700 GHz + 3,068,718,694 instructions:u # 1.64 insn per cycle + 1.104959937 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 3364) (512y: 0) (512z: 1959) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512z_f_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512z_f_inl1/runTest.exe [ PASSED ] 3 tests. ========================================================================= From b41e672ae39dc2ee87ae3d9b3f0e37e0a3e9bbcd Mon Sep 17 00:00:00 2001 From: Andrea Valassi Date: Wed, 1 Dec 2021 11:09:59 +0100 Subject: [PATCH 16/34] [klas5X] in eemumu manu use "mkdir -p" to avoid errors if dir exists and thus allow parallel "make -j" --- epochX/cudacpp/ee_mumu/SubProcesses/Makefile | 12 ++++++------ epochX/cudacpp/ee_mumu/src/Makefile | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/epochX/cudacpp/ee_mumu/SubProcesses/Makefile b/epochX/cudacpp/ee_mumu/SubProcesses/Makefile index 69a9a4dbe4..e57c3f2024 100644 --- a/epochX/cudacpp/ee_mumu/SubProcesses/Makefile +++ b/epochX/cudacpp/ee_mumu/SubProcesses/Makefile @@ -250,7 +250,7 @@ all.$(TAG): ../../src/$(BUILDDIR)/.build.$(TAG) $(BUILDDIR)/.build.$(TAG) $(cu_m override oldtags=`find $(BUILDDIR) -maxdepth 1 -name '.build.*' ! -name '.build.$(TAG)'` $(BUILDDIR)/.build.$(TAG): - @if [ ! -d $(BUILDDIR) ]; then echo "mkdir $(BUILDDIR)"; mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then echo "mkdir -p $(BUILDDIR)"; mkdir -p $(BUILDDIR); fi @if [ "$(oldtags)" != "" ]; then echo -e "Cannot build for tag=$(TAG) as old builds exist for other tags:\n$(oldtags)\nPlease run 'make clean' first\nIf 'make clean' is not enough: run 'make clean USEBUILDDIR=1 AVX=$(AVX) FPTYPE=$(FPTYPE)' or 'make cleanall'"; exit 1; fi @touch $(BUILDDIR)/.build.$(TAG) @@ -266,28 +266,28 @@ $(LIBDIR)/lib$(MODELLIB).a: ../../src/*.h ../../src/*.cc $(MAKE) -C ../../src $(MAKEDEBUG) $(BUILDDIR)/gcheck_sa.o: gcheck_sa.cu *.h ../../src/*.h # ../../src/*.cu - @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi $(NVCC) $(CPPFLAGS) $(CUFLAGS) -c $< -o $@ $(BUILDDIR)/CPPProcess.o : ../../src/HelAmps_sm.cc $(BUILDDIR)/gCPPProcess.o : ../../src/HelAmps_sm.cc $(BUILDDIR)/%.o : %.cu *.h ../../src/*.h - @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi $(NVCC) $(CPPFLAGS) $(CUFLAGS) -c $< -o $@ # Apply special build flags only to CPPProcess.cc (-flto) #$(BUILDDIR)/CPPProcess.o : CPPProcess.cc *.h ../../src/*.h -# @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi +# @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi # $(CXX) $(CPPFLAGS) $(CXXFLAGS) -flto $(CUINC) -c $< -o $@ ### Apply special build flags only to CPPProcess.cc (AVXFLAGS) ###$(BUILDDIR)/CPPProcess.o : CPPProcess.cc *.h ../../src/*.h -### @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi +### @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi ### $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(AVXFLAGS) $(CUINC) -c $< -o $@ $(BUILDDIR)/%.o : %.cc *.h ../../src/*.h - @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(CUINC) -c $< -o $@ $(cu_main): $(BUILDDIR)/gcheck_sa.o $(LIBDIR)/lib$(MODELLIB).a $(cu_objects) diff --git a/epochX/cudacpp/ee_mumu/src/Makefile b/epochX/cudacpp/ee_mumu/src/Makefile index ce74470562..6e6b6a0c39 100644 --- a/epochX/cudacpp/ee_mumu/src/Makefile +++ b/epochX/cudacpp/ee_mumu/src/Makefile @@ -160,7 +160,7 @@ all.$(TAG): $(BUILDDIR)/.build.$(TAG) $(target) override oldtags=`find $(BUILDDIR) -maxdepth 1 -name '.build.*' ! -name '.build.$(TAG)'` $(BUILDDIR)/.build.$(TAG): - @if [ ! -d $(BUILDDIR) ]; then echo "mkdir $(BUILDDIR)"; mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then echo "mkdir -p $(BUILDDIR)"; mkdir -p $(BUILDDIR); fi @if [ "$(oldtags)" != "" ]; then echo -e "Cannot build for tag=$(TAG) as old builds exist for other tags:\n$(oldtags)\nPlease run 'make clean' first\nIf 'make clean' is not enough: run 'make clean USEBUILDDIR=1 AVX=$(AVX) FPTYPE=$(FPTYPE)' or 'make cleanall'"; exit 1; fi @touch $(BUILDDIR)/.build.$(TAG) @@ -171,11 +171,11 @@ debug: $(target) # NB: cuda includes are needed in the C++ code for curand.h $(BUILDDIR)/%.o : %.cc *.h - @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(CUINC) -c $< -o $@ #$(BUILDDIR)/%.o : %.cu *.h -# @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi +# @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi # $(NVCC) $(CPPFLAGS) $(CUFLAGS) -c $< -o $@ $(target): $(cxx_objects) $(cu_objects) From 6275826198738476c93418e7794090300519203b Mon Sep 17 00:00:00 2001 From: Andrea Valassi Date: Wed, 1 Dec 2021 11:12:19 +0100 Subject: [PATCH 17/34] [klas5X] in CODEGEN use "mkdir -p" to avoid errors if dir exists and thus allow parallel "make -j" --- .../madgraph/iolibs/template_files/gpu/Makefile | 12 ++++++------ .../madgraph/iolibs/template_files/gpu/Makefile_src | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/Makefile b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/Makefile index 69ee05a2e0..102323ed4b 100644 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/Makefile +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/Makefile @@ -250,7 +250,7 @@ all.$(TAG): ../../src/$(BUILDDIR)/.build.$(TAG) $(BUILDDIR)/.build.$(TAG) $(cu_m override oldtags=`find $(BUILDDIR) -maxdepth 1 -name '.build.*' ! -name '.build.$(TAG)'` $(BUILDDIR)/.build.$(TAG): - @if [ ! -d $(BUILDDIR) ]; then echo "mkdir $(BUILDDIR)"; mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then echo "mkdir -p $(BUILDDIR)"; mkdir -p $(BUILDDIR); fi @if [ "$(oldtags)" != "" ]; then echo -e "Cannot build for tag=$(TAG) as old builds exist for other tags:\n$(oldtags)\nPlease run 'make clean' first\nIf 'make clean' is not enough: run 'make clean USEBUILDDIR=1 AVX=$(AVX) FPTYPE=$(FPTYPE)' or 'make cleanall'"; exit 1; fi @touch $(BUILDDIR)/.build.$(TAG) @@ -266,28 +266,28 @@ $(LIBDIR)/lib$(MODELLIB).a: ../../src/*.h ../../src/*.cc $(MAKE) -C ../../src $(MAKEDEBUG) $(BUILDDIR)/gcheck_sa.o: gcheck_sa.cu *.h ../../src/*.h # ../../src/*.cu - @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi $(NVCC) $(CPPFLAGS) $(CUFLAGS) -c $< -o $@ $(BUILDDIR)/CPPProcess.o : ../../src/HelAmps_sm.cc $(BUILDDIR)/gCPPProcess.o : ../../src/HelAmps_sm.cc $(BUILDDIR)/%%.o : %%.cu *.h ../../src/*.h - @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi $(NVCC) $(CPPFLAGS) $(CUFLAGS) -c $< -o $@ # Apply special build flags only to CPPProcess.cc (-flto) #$(BUILDDIR)/CPPProcess.o : CPPProcess.cc *.h ../../src/*.h -# @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi +# @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi # $(CXX) $(CPPFLAGS) $(CXXFLAGS) -flto $(CUINC) -c $< -o $@ ### Apply special build flags only to CPPProcess.cc (AVXFLAGS) ###$(BUILDDIR)/CPPProcess.o : CPPProcess.cc *.h ../../src/*.h -### @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi +### @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi ### $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(AVXFLAGS) $(CUINC) -c $< -o $@ $(BUILDDIR)/%%.o : %%.cc *.h ../../src/*.h - @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(CUINC) -c $< -o $@ $(cu_main): $(BUILDDIR)/gcheck_sa.o $(LIBDIR)/lib$(MODELLIB).a $(cu_objects) diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/Makefile_src b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/Makefile_src index 828333ebad..6faee6c69c 100644 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/Makefile_src +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/Makefile_src @@ -160,7 +160,7 @@ all.$(TAG): $(BUILDDIR)/.build.$(TAG) $(target) override oldtags=`find $(BUILDDIR) -maxdepth 1 -name '.build.*' ! -name '.build.$(TAG)'` $(BUILDDIR)/.build.$(TAG): - @if [ ! -d $(BUILDDIR) ]; then echo "mkdir $(BUILDDIR)"; mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then echo "mkdir -p $(BUILDDIR)"; mkdir -p $(BUILDDIR); fi @if [ "$(oldtags)" != "" ]; then echo -e "Cannot build for tag=$(TAG) as old builds exist for other tags:\n$(oldtags)\nPlease run 'make clean' first\nIf 'make clean' is not enough: run 'make clean USEBUILDDIR=1 AVX=$(AVX) FPTYPE=$(FPTYPE)' or 'make cleanall'"; exit 1; fi @touch $(BUILDDIR)/.build.$(TAG) @@ -171,11 +171,11 @@ debug: $(target) # NB: cuda includes are needed in the C++ code for curand.h $(BUILDDIR)/%%.o : %%.cc *.h - @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(CUINC) -c $< -o $@ #$(BUILDDIR)/%%.o : %%.cu *.h -# @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi +# @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi # $(NVCC) $(CPPFLAGS) $(CUFLAGS) -c $< -o $@ $(target): $(cxx_objects) $(cu_objects) From 52efa816f10aa5afd210bd4703c7dd0d56e04b5f Mon Sep 17 00:00:00 2001 From: Andrea Valassi Date: Wed, 1 Dec 2021 11:13:10 +0100 Subject: [PATCH 18/34] [klas5X] regenerate eemumu and ggtt auto with "mkdir -p" to allow parallel make --- .../ee_mumu.auto/CODEGEN_cudacpp_ee_mumu_log.txt | 8 ++++---- epochX/cudacpp/ee_mumu.auto/SubProcesses/Makefile | 12 ++++++------ epochX/cudacpp/ee_mumu.auto/src/Makefile | 6 +++--- .../cudacpp/gg_tt.auto/CODEGEN_cudacpp_gg_tt_log.txt | 8 ++++---- epochX/cudacpp/gg_tt.auto/SubProcesses/Makefile | 12 ++++++------ epochX/cudacpp/gg_tt.auto/src/Makefile | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/epochX/cudacpp/ee_mumu.auto/CODEGEN_cudacpp_ee_mumu_log.txt b/epochX/cudacpp/ee_mumu.auto/CODEGEN_cudacpp_ee_mumu_log.txt index 9c07f0f53d..b3728dde47 100644 --- a/epochX/cudacpp/ee_mumu.auto/CODEGEN_cudacpp_ee_mumu_log.txt +++ b/epochX/cudacpp/ee_mumu.auto/CODEGEN_cudacpp_ee_mumu_log.txt @@ -50,7 +50,7 @@ generate e+ e- > mu+ mu- No model currently active, so we import the Standard Model INFO: load particles INFO: load vertices -DEBUG: model prefixing takes 0.0068590641021728516  +DEBUG: model prefixing takes 0.006864309310913086  INFO: Restrict model sm with file models/sm/restrict_default.dat . INFO: Change particles name to pass to MG5 convention DEBUG: True [misc.py at line 2192]  @@ -121,6 +121,6 @@ INFO: /data/avalassi/GPU2020/MG5aMC/2.7.0_gpu/CODEGEN_cudacpp_ee_mumu/src/. and DEBUG: Entering PLUGIN_ProcessExporter.finalize [output.py at line 154]  quit -real 0m18.430s -user 0m0.923s -sys 0m0.138s +real 0m4.641s +user 0m0.928s +sys 0m0.135s diff --git a/epochX/cudacpp/ee_mumu.auto/SubProcesses/Makefile b/epochX/cudacpp/ee_mumu.auto/SubProcesses/Makefile index 69a9a4dbe4..e57c3f2024 100644 --- a/epochX/cudacpp/ee_mumu.auto/SubProcesses/Makefile +++ b/epochX/cudacpp/ee_mumu.auto/SubProcesses/Makefile @@ -250,7 +250,7 @@ all.$(TAG): ../../src/$(BUILDDIR)/.build.$(TAG) $(BUILDDIR)/.build.$(TAG) $(cu_m override oldtags=`find $(BUILDDIR) -maxdepth 1 -name '.build.*' ! -name '.build.$(TAG)'` $(BUILDDIR)/.build.$(TAG): - @if [ ! -d $(BUILDDIR) ]; then echo "mkdir $(BUILDDIR)"; mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then echo "mkdir -p $(BUILDDIR)"; mkdir -p $(BUILDDIR); fi @if [ "$(oldtags)" != "" ]; then echo -e "Cannot build for tag=$(TAG) as old builds exist for other tags:\n$(oldtags)\nPlease run 'make clean' first\nIf 'make clean' is not enough: run 'make clean USEBUILDDIR=1 AVX=$(AVX) FPTYPE=$(FPTYPE)' or 'make cleanall'"; exit 1; fi @touch $(BUILDDIR)/.build.$(TAG) @@ -266,28 +266,28 @@ $(LIBDIR)/lib$(MODELLIB).a: ../../src/*.h ../../src/*.cc $(MAKE) -C ../../src $(MAKEDEBUG) $(BUILDDIR)/gcheck_sa.o: gcheck_sa.cu *.h ../../src/*.h # ../../src/*.cu - @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi $(NVCC) $(CPPFLAGS) $(CUFLAGS) -c $< -o $@ $(BUILDDIR)/CPPProcess.o : ../../src/HelAmps_sm.cc $(BUILDDIR)/gCPPProcess.o : ../../src/HelAmps_sm.cc $(BUILDDIR)/%.o : %.cu *.h ../../src/*.h - @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi $(NVCC) $(CPPFLAGS) $(CUFLAGS) -c $< -o $@ # Apply special build flags only to CPPProcess.cc (-flto) #$(BUILDDIR)/CPPProcess.o : CPPProcess.cc *.h ../../src/*.h -# @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi +# @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi # $(CXX) $(CPPFLAGS) $(CXXFLAGS) -flto $(CUINC) -c $< -o $@ ### Apply special build flags only to CPPProcess.cc (AVXFLAGS) ###$(BUILDDIR)/CPPProcess.o : CPPProcess.cc *.h ../../src/*.h -### @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi +### @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi ### $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(AVXFLAGS) $(CUINC) -c $< -o $@ $(BUILDDIR)/%.o : %.cc *.h ../../src/*.h - @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(CUINC) -c $< -o $@ $(cu_main): $(BUILDDIR)/gcheck_sa.o $(LIBDIR)/lib$(MODELLIB).a $(cu_objects) diff --git a/epochX/cudacpp/ee_mumu.auto/src/Makefile b/epochX/cudacpp/ee_mumu.auto/src/Makefile index ce74470562..6e6b6a0c39 100644 --- a/epochX/cudacpp/ee_mumu.auto/src/Makefile +++ b/epochX/cudacpp/ee_mumu.auto/src/Makefile @@ -160,7 +160,7 @@ all.$(TAG): $(BUILDDIR)/.build.$(TAG) $(target) override oldtags=`find $(BUILDDIR) -maxdepth 1 -name '.build.*' ! -name '.build.$(TAG)'` $(BUILDDIR)/.build.$(TAG): - @if [ ! -d $(BUILDDIR) ]; then echo "mkdir $(BUILDDIR)"; mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then echo "mkdir -p $(BUILDDIR)"; mkdir -p $(BUILDDIR); fi @if [ "$(oldtags)" != "" ]; then echo -e "Cannot build for tag=$(TAG) as old builds exist for other tags:\n$(oldtags)\nPlease run 'make clean' first\nIf 'make clean' is not enough: run 'make clean USEBUILDDIR=1 AVX=$(AVX) FPTYPE=$(FPTYPE)' or 'make cleanall'"; exit 1; fi @touch $(BUILDDIR)/.build.$(TAG) @@ -171,11 +171,11 @@ debug: $(target) # NB: cuda includes are needed in the C++ code for curand.h $(BUILDDIR)/%.o : %.cc *.h - @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(CUINC) -c $< -o $@ #$(BUILDDIR)/%.o : %.cu *.h -# @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi +# @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi # $(NVCC) $(CPPFLAGS) $(CUFLAGS) -c $< -o $@ $(target): $(cxx_objects) $(cu_objects) diff --git a/epochX/cudacpp/gg_tt.auto/CODEGEN_cudacpp_gg_tt_log.txt b/epochX/cudacpp/gg_tt.auto/CODEGEN_cudacpp_gg_tt_log.txt index 0d553f454d..64e46f0fa7 100644 --- a/epochX/cudacpp/gg_tt.auto/CODEGEN_cudacpp_gg_tt_log.txt +++ b/epochX/cudacpp/gg_tt.auto/CODEGEN_cudacpp_gg_tt_log.txt @@ -50,7 +50,7 @@ generate g g > t t~ No model currently active, so we import the Standard Model INFO: load particles INFO: load vertices -DEBUG: model prefixing takes 0.006850719451904297  +DEBUG: model prefixing takes 0.006934404373168945  INFO: Restrict model sm with file models/sm/restrict_default.dat . INFO: Change particles name to pass to MG5 convention DEBUG: True [misc.py at line 2192]  @@ -118,6 +118,6 @@ INFO: /data/avalassi/GPU2020/MG5aMC/2.7.0_gpu/CODEGEN_cudacpp_gg_tt/src/. and /d DEBUG: Entering PLUGIN_ProcessExporter.finalize [output.py at line 154]  quit -real 0m3.666s -user 0m0.772s -sys 0m0.133s +real 0m3.842s +user 0m0.806s +sys 0m0.116s diff --git a/epochX/cudacpp/gg_tt.auto/SubProcesses/Makefile b/epochX/cudacpp/gg_tt.auto/SubProcesses/Makefile index 69a9a4dbe4..e57c3f2024 100644 --- a/epochX/cudacpp/gg_tt.auto/SubProcesses/Makefile +++ b/epochX/cudacpp/gg_tt.auto/SubProcesses/Makefile @@ -250,7 +250,7 @@ all.$(TAG): ../../src/$(BUILDDIR)/.build.$(TAG) $(BUILDDIR)/.build.$(TAG) $(cu_m override oldtags=`find $(BUILDDIR) -maxdepth 1 -name '.build.*' ! -name '.build.$(TAG)'` $(BUILDDIR)/.build.$(TAG): - @if [ ! -d $(BUILDDIR) ]; then echo "mkdir $(BUILDDIR)"; mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then echo "mkdir -p $(BUILDDIR)"; mkdir -p $(BUILDDIR); fi @if [ "$(oldtags)" != "" ]; then echo -e "Cannot build for tag=$(TAG) as old builds exist for other tags:\n$(oldtags)\nPlease run 'make clean' first\nIf 'make clean' is not enough: run 'make clean USEBUILDDIR=1 AVX=$(AVX) FPTYPE=$(FPTYPE)' or 'make cleanall'"; exit 1; fi @touch $(BUILDDIR)/.build.$(TAG) @@ -266,28 +266,28 @@ $(LIBDIR)/lib$(MODELLIB).a: ../../src/*.h ../../src/*.cc $(MAKE) -C ../../src $(MAKEDEBUG) $(BUILDDIR)/gcheck_sa.o: gcheck_sa.cu *.h ../../src/*.h # ../../src/*.cu - @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi $(NVCC) $(CPPFLAGS) $(CUFLAGS) -c $< -o $@ $(BUILDDIR)/CPPProcess.o : ../../src/HelAmps_sm.cc $(BUILDDIR)/gCPPProcess.o : ../../src/HelAmps_sm.cc $(BUILDDIR)/%.o : %.cu *.h ../../src/*.h - @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi $(NVCC) $(CPPFLAGS) $(CUFLAGS) -c $< -o $@ # Apply special build flags only to CPPProcess.cc (-flto) #$(BUILDDIR)/CPPProcess.o : CPPProcess.cc *.h ../../src/*.h -# @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi +# @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi # $(CXX) $(CPPFLAGS) $(CXXFLAGS) -flto $(CUINC) -c $< -o $@ ### Apply special build flags only to CPPProcess.cc (AVXFLAGS) ###$(BUILDDIR)/CPPProcess.o : CPPProcess.cc *.h ../../src/*.h -### @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi +### @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi ### $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(AVXFLAGS) $(CUINC) -c $< -o $@ $(BUILDDIR)/%.o : %.cc *.h ../../src/*.h - @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(CUINC) -c $< -o $@ $(cu_main): $(BUILDDIR)/gcheck_sa.o $(LIBDIR)/lib$(MODELLIB).a $(cu_objects) diff --git a/epochX/cudacpp/gg_tt.auto/src/Makefile b/epochX/cudacpp/gg_tt.auto/src/Makefile index ce74470562..6e6b6a0c39 100644 --- a/epochX/cudacpp/gg_tt.auto/src/Makefile +++ b/epochX/cudacpp/gg_tt.auto/src/Makefile @@ -160,7 +160,7 @@ all.$(TAG): $(BUILDDIR)/.build.$(TAG) $(target) override oldtags=`find $(BUILDDIR) -maxdepth 1 -name '.build.*' ! -name '.build.$(TAG)'` $(BUILDDIR)/.build.$(TAG): - @if [ ! -d $(BUILDDIR) ]; then echo "mkdir $(BUILDDIR)"; mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then echo "mkdir -p $(BUILDDIR)"; mkdir -p $(BUILDDIR); fi @if [ "$(oldtags)" != "" ]; then echo -e "Cannot build for tag=$(TAG) as old builds exist for other tags:\n$(oldtags)\nPlease run 'make clean' first\nIf 'make clean' is not enough: run 'make clean USEBUILDDIR=1 AVX=$(AVX) FPTYPE=$(FPTYPE)' or 'make cleanall'"; exit 1; fi @touch $(BUILDDIR)/.build.$(TAG) @@ -171,11 +171,11 @@ debug: $(target) # NB: cuda includes are needed in the C++ code for curand.h $(BUILDDIR)/%.o : %.cc *.h - @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(CUINC) -c $< -o $@ #$(BUILDDIR)/%.o : %.cu *.h -# @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi +# @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi # $(NVCC) $(CPPFLAGS) $(CUFLAGS) -c $< -o $@ $(target): $(cxx_objects) $(cu_objects) From 573ace24aaa649ecfde1edc20635f3cf09e22298 Mon Sep 17 00:00:00 2001 From: Andrea Valassi Date: Wed, 1 Dec 2021 11:37:20 +0100 Subject: [PATCH 19/34] [klas5X] use parallel 'make -j' in throughputX script --- epochX/cudacpp/tput/throughputX.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/epochX/cudacpp/tput/throughputX.sh b/epochX/cudacpp/tput/throughputX.sh index 985614c489..b966516931 100755 --- a/epochX/cudacpp/tput/throughputX.sh +++ b/epochX/cudacpp/tput/throughputX.sh @@ -205,12 +205,12 @@ for suff in $suffs; do export HELINL=$helinl for fptype in $fptypes; do export FPTYPE=$fptype - make AVX=none; echo - if [ "${avxall}" == "1" ]; then make AVX=sse4; echo; fi - if [ "${avxall}" == "1" ]; then make AVX=avx2; echo; fi + make -j AVX=none; echo + if [ "${avxall}" == "1" ]; then make -j AVX=sse4; echo; fi + if [ "${avxall}" == "1" ]; then make -j AVX=avx2; echo; fi if [ "$(grep -m1 -c avx512vl /proc/cpuinfo)" == "1" ]; then # skip avx512 if not supported! - if [ "${cpp}" == "1" ]; then make AVX=512y; echo; fi # use 512y as C++ ref even if avx2 is faster on clang - if [ "${avxall}" == "1" ]; then make AVX=512z; echo; fi + if [ "${cpp}" == "1" ]; then make -j AVX=512y; echo; fi # use 512y as C++ ref even if avx2 is faster on clang + if [ "${avxall}" == "1" ]; then make -j AVX=512z; echo; fi fi done done From 38fee9bcea33c392b8447f02f1c088c6941fb466 Mon Sep 17 00:00:00 2001 From: Andrea Valassi Date: Wed, 1 Dec 2021 11:39:07 +0100 Subject: [PATCH 20/34] [klas5X] regenerate ggttgg auto and run all tests - all ok, essentially the same performance (next commit) --- .../CODEGEN_cudacpp_gg_ttgg_log.txt | 40 ++-- .../gg_ttgg.auto/SubProcesses/Makefile | 12 +- .../gg_ttgg.auto/SubProcesses/Memory.h | 19 +- .../P1_Sigma_sm_gg_ttxgg/CPPProcess.cc | 82 +++++--- .../P1_Sigma_sm_gg_ttxgg/CPPProcess.h | 14 +- .../P1_Sigma_sm_gg_ttxgg/check_sa.cc | 35 +--- .../gg_ttgg.auto/SubProcesses/runTest.cc | 36 ++-- .../gg_ttgg.auto/SubProcesses/testxxx.cc | 8 +- epochX/cudacpp/gg_ttgg.auto/src/HelAmps_sm.cc | 183 ++++++++++++++---- epochX/cudacpp/gg_ttgg.auto/src/HelAmps_sm.h | 20 +- epochX/cudacpp/gg_ttgg.auto/src/Makefile | 6 +- .../cudacpp/gg_ttgg.auto/src/mgOnGpuConfig.h | 84 +++++--- .../cudacpp/gg_ttgg.auto/src/mgOnGpuVectors.h | 16 +- epochX/cudacpp/gg_ttgg.auto/src/rambo.cc | 6 +- epochX/cudacpp/gg_ttgg.auto/src/rambo.h | 6 +- 15 files changed, 364 insertions(+), 203 deletions(-) diff --git a/epochX/cudacpp/gg_ttgg.auto/CODEGEN_cudacpp_gg_ttgg_log.txt b/epochX/cudacpp/gg_ttgg.auto/CODEGEN_cudacpp_gg_ttgg_log.txt index b2ae4b6185..8195fc5150 100644 --- a/epochX/cudacpp/gg_ttgg.auto/CODEGEN_cudacpp_gg_ttgg_log.txt +++ b/epochX/cudacpp/gg_ttgg.auto/CODEGEN_cudacpp_gg_ttgg_log.txt @@ -50,7 +50,7 @@ generate g g > t t~ g g No model currently active, so we import the Standard Model INFO: load particles INFO: load vertices -DEBUG: model prefixing takes 0.0068645477294921875  +DEBUG: model prefixing takes 0.006870269775390625  INFO: Restrict model sm with file models/sm/restrict_default.dat . INFO: Change particles name to pass to MG5 convention DEBUG: True [misc.py at line 2192]  @@ -67,7 +67,7 @@ INFO: Please specify coupling orders to bypass this step. INFO: Trying coupling order WEIGHTED<=4: WEIGTHED IS QCD+2*QED INFO: Trying process: g g > t t~ g g WEIGHTED<=4 @1 INFO: Process has 123 diagrams -1 processes with 123 diagrams generated in 0.208 s +1 processes with 123 diagrams generated in 0.207 s Total: 1 processes with 123 diagrams output standalone_cudacpp CODEGEN_cudacpp_gg_ttgg Output will be done with PLUGIN: CUDACPP_SA_OUTPUT @@ -82,25 +82,25 @@ INFO: Processing color information for process: g g > t t~ g g @1 DEBUG: type(subproc_group)= [output.py at line 139]  DEBUG: type(fortran_model)= [output.py at line 140]  INFO: Creating files in directory /data/avalassi/GPU2020/MG5aMC/2.7.0_gpu/CODEGEN_cudacpp_gg_ttgg/SubProcesses/P1_Sigma_sm_gg_ttxgg -DEBUG: Entering PLUGIN_OneProcessExporter.generate_process_files [model_handling.py at line 886]  +DEBUG: Entering PLUGIN_OneProcessExporter.generate_process_files [model_handling.py at line 893]  FileWriter for /data/avalassi/GPU2020/MG5aMC/2.7.0_gpu/CODEGEN_cudacpp_gg_ttgg/SubProcesses/P1_Sigma_sm_gg_ttxgg/./CPPProcess.h -DEBUG: Entering PLUGIN_OneProcessExporter.write_process_h_file [model_handling.py at line 935]  +DEBUG: Entering PLUGIN_OneProcessExporter.write_process_h_file [model_handling.py at line 942]  FileWriter for /data/avalassi/GPU2020/MG5aMC/2.7.0_gpu/CODEGEN_cudacpp_gg_ttgg/SubProcesses/P1_Sigma_sm_gg_ttxgg/./CPPProcess.cc -DEBUG: Entering PLUGIN_OneProcessExporter.write_process_cc_file [model_handling.py at line 957]  -DEBUG: call =  vxxxxx(allmomenta,m_pars->%s, cHel[ihel][%d],%+d, w_sv[%d], %d); [model_handling.py at line 1234]  -DEBUG: ('ZERO', 0, -1, 0, 0) [model_handling.py at line 1235]  -DEBUG: call =  vxxxxx(allmomenta,m_pars->%s, cHel[ihel][%d],%+d, w_sv[%d], %d); [model_handling.py at line 1234]  -DEBUG: ('ZERO', 1, -1, 1, 1) [model_handling.py at line 1235]  -DEBUG: call =  vxxxxx(allmomenta,m_pars->%s, cHel[ihel][%d],%+d, w_sv[%d], %d); [model_handling.py at line 1234]  -DEBUG: ('ZERO', 4, 1, 4, 4) [model_handling.py at line 1235]  -DEBUG: call =  vxxxxx(allmomenta,m_pars->%s, cHel[ihel][%d],%+d, w_sv[%d], %d); [model_handling.py at line 1234]  -DEBUG: ('ZERO', 5, 1, 5, 5) [model_handling.py at line 1235]  +DEBUG: Entering PLUGIN_OneProcessExporter.write_process_cc_file [model_handling.py at line 964]  +DEBUG: call =  vxxxxx(allmomenta,m_pars->%s, cHel[ihel][%d],%+d, w_sv[%d], %d); [model_handling.py at line 1241]  +DEBUG: ('ZERO', 0, -1, 0, 0) [model_handling.py at line 1242]  +DEBUG: call =  vxxxxx(allmomenta,m_pars->%s, cHel[ihel][%d],%+d, w_sv[%d], %d); [model_handling.py at line 1241]  +DEBUG: ('ZERO', 1, -1, 1, 1) [model_handling.py at line 1242]  +DEBUG: call =  vxxxxx(allmomenta,m_pars->%s, cHel[ihel][%d],%+d, w_sv[%d], %d); [model_handling.py at line 1241]  +DEBUG: ('ZERO', 4, 1, 4, 4) [model_handling.py at line 1242]  +DEBUG: call =  vxxxxx(allmomenta,m_pars->%s, cHel[ihel][%d],%+d, w_sv[%d], %d); [model_handling.py at line 1241]  +DEBUG: ('ZERO', 5, 1, 5, 5) [model_handling.py at line 1242]  DEBUG: only one Matrix-element supported?  INFO: Created files CPPProcess.h and CPPProcess.cc in directory /data/avalassi/GPU2020/MG5aMC/2.7.0_gpu/CODEGEN_cudacpp_gg_ttgg/SubProcesses/P1_Sigma_sm_gg_ttxgg/. -DEBUG: Entering PLUGIN_OneProcessExporter.edit_check_sa [model_handling.py at line 900]  -DEBUG: Entering PLUGIN_OneProcessExporter.edit_mgonGPU [model_handling.py at line 915]  -DEBUG: Entering PLUGIN_OneProcessExporter.edit_processidfile [model_handling.py at line 922]  -Generated helas calls for 1 subprocesses (123 diagrams) in 0.560 s +DEBUG: Entering PLUGIN_OneProcessExporter.edit_check_sa [model_handling.py at line 907]  +DEBUG: Entering PLUGIN_OneProcessExporter.edit_mgonGPU [model_handling.py at line 922]  +DEBUG: Entering PLUGIN_OneProcessExporter.edit_processidfile [model_handling.py at line 929]  +Generated helas calls for 1 subprocesses (123 diagrams) in 0.558 s DEBUG: Entering PLUGIN_ProcessExporter.convert_model (create the model) [output.py at line 145]  ALOHA: aloha creates routines (starting by VVV1) VVV1 @@ -138,6 +138,6 @@ INFO: /data/avalassi/GPU2020/MG5aMC/2.7.0_gpu/CODEGEN_cudacpp_gg_ttgg/src/. and DEBUG: Entering PLUGIN_ProcessExporter.finalize [output.py at line 154]  quit -real 0m5.324s -user 0m1.954s -sys 0m0.162s +real 0m5.019s +user 0m1.967s +sys 0m0.149s diff --git a/epochX/cudacpp/gg_ttgg.auto/SubProcesses/Makefile b/epochX/cudacpp/gg_ttgg.auto/SubProcesses/Makefile index 69a9a4dbe4..e57c3f2024 100644 --- a/epochX/cudacpp/gg_ttgg.auto/SubProcesses/Makefile +++ b/epochX/cudacpp/gg_ttgg.auto/SubProcesses/Makefile @@ -250,7 +250,7 @@ all.$(TAG): ../../src/$(BUILDDIR)/.build.$(TAG) $(BUILDDIR)/.build.$(TAG) $(cu_m override oldtags=`find $(BUILDDIR) -maxdepth 1 -name '.build.*' ! -name '.build.$(TAG)'` $(BUILDDIR)/.build.$(TAG): - @if [ ! -d $(BUILDDIR) ]; then echo "mkdir $(BUILDDIR)"; mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then echo "mkdir -p $(BUILDDIR)"; mkdir -p $(BUILDDIR); fi @if [ "$(oldtags)" != "" ]; then echo -e "Cannot build for tag=$(TAG) as old builds exist for other tags:\n$(oldtags)\nPlease run 'make clean' first\nIf 'make clean' is not enough: run 'make clean USEBUILDDIR=1 AVX=$(AVX) FPTYPE=$(FPTYPE)' or 'make cleanall'"; exit 1; fi @touch $(BUILDDIR)/.build.$(TAG) @@ -266,28 +266,28 @@ $(LIBDIR)/lib$(MODELLIB).a: ../../src/*.h ../../src/*.cc $(MAKE) -C ../../src $(MAKEDEBUG) $(BUILDDIR)/gcheck_sa.o: gcheck_sa.cu *.h ../../src/*.h # ../../src/*.cu - @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi $(NVCC) $(CPPFLAGS) $(CUFLAGS) -c $< -o $@ $(BUILDDIR)/CPPProcess.o : ../../src/HelAmps_sm.cc $(BUILDDIR)/gCPPProcess.o : ../../src/HelAmps_sm.cc $(BUILDDIR)/%.o : %.cu *.h ../../src/*.h - @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi $(NVCC) $(CPPFLAGS) $(CUFLAGS) -c $< -o $@ # Apply special build flags only to CPPProcess.cc (-flto) #$(BUILDDIR)/CPPProcess.o : CPPProcess.cc *.h ../../src/*.h -# @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi +# @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi # $(CXX) $(CPPFLAGS) $(CXXFLAGS) -flto $(CUINC) -c $< -o $@ ### Apply special build flags only to CPPProcess.cc (AVXFLAGS) ###$(BUILDDIR)/CPPProcess.o : CPPProcess.cc *.h ../../src/*.h -### @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi +### @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi ### $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(AVXFLAGS) $(CUINC) -c $< -o $@ $(BUILDDIR)/%.o : %.cc *.h ../../src/*.h - @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(CUINC) -c $< -o $@ $(cu_main): $(BUILDDIR)/gcheck_sa.o $(LIBDIR)/lib$(MODELLIB).a $(cu_objects) diff --git a/epochX/cudacpp/gg_ttgg.auto/SubProcesses/Memory.h b/epochX/cudacpp/gg_ttgg.auto/SubProcesses/Memory.h index d7d1c7d4da..a9c08d3dd0 100644 --- a/epochX/cudacpp/gg_ttgg.auto/SubProcesses/Memory.h +++ b/epochX/cudacpp/gg_ttgg.auto/SubProcesses/Memory.h @@ -1,8 +1,9 @@ /* - * memory.h + * Memory.h * * Created on: 19.11.2020 * Author: shageboeck + * Modified: avalassi (vectors, alignment) */ #ifndef MEMORY_H @@ -46,13 +47,25 @@ std::unique_ptr> hstMakeUnique(std::size_t N) { #else +template +struct CppHstDeleter { + void operator()(T* mem) { + ::operator delete( mem, std::align_val_t{ mgOnGpu::cppAlign } ); + } +}; + template inline -std::unique_ptr hstMakeUnique(std::size_t N) { return std::unique_ptr{ new T[N]() }; }; +std::unique_ptr> hstMakeUnique(std::size_t N) { + // See https://www.bfilipek.com/2019/08/newnew-align.html#requesting-different-alignment + return std::unique_ptr>{ new( std::align_val_t{ mgOnGpu::cppAlign } ) T[N]() }; +}; #ifdef MGONGPU_CPPSIMD template<> inline -std::unique_ptr hstMakeUnique(std::size_t N) { return std::unique_ptr{ new fptype_v[N/neppV]() }; }; +std::unique_ptr> hstMakeUnique(std::size_t N) { + return std::unique_ptr>{ new( std::align_val_t{ mgOnGpu::cppAlign } ) fptype_v[N/neppV]() }; +}; #endif diff --git a/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/CPPProcess.cc b/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/CPPProcess.cc index 164b8af4d7..c4bd7eeb18 100644 --- a/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/CPPProcess.cc +++ b/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/CPPProcess.cc @@ -71,10 +71,10 @@ namespace Proc __device__ INLINE void calculate_wavefunctions( int ihel, - const fptype_sv* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM - fptype_sv* allMEs // output: allMEs[npagM][neppM], final |M|^2 averaged over helicities + const fptype* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM + fptype* allMEs // output: allMEs[nevt], |M|^2 running_sum_over_helicities #ifndef __CUDACC__ - , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) #endif ) //ALWAYS_INLINE // attributes are not permitted in a function definition @@ -99,6 +99,9 @@ namespace Proc // === Calculate wavefunctions and amplitudes for all diagrams in all processes - Loop over nevt events === #ifndef __CUDACC__ const int npagV = nevt / neppV; +#ifdef MGONGPU_CPPSIMD + const bool isAligned_allMEs = ( (size_t)(allMEs) % mgOnGpu::cppAlign == 0 ); // require SIMD-friendly alignment by at least neppV*sizeof(fptype) +#endif // ** START LOOP ON IPAGV ** #ifdef _OPENMP // (NB gcc9 or higher, or clang, is required) @@ -106,7 +109,11 @@ namespace Proc // - shared: as the name says // - private: give each thread its own copy, without initialising // - firstprivate: give each thread its own copy, and initialise with value from outside +#ifdef MGONGPU_CPPSIMD +#pragma omp parallel for default(none) shared(allmomenta,allMEs,cHel,cIPC,cIPD,ihel,npagV,isAligned_allMEs) private (amp_sv,w_sv,jamp_sv) +#else #pragma omp parallel for default(none) shared(allmomenta,allMEs,cHel,cIPC,cIPD,ihel,npagV) private (amp_sv,w_sv,jamp_sv) +#endif #endif for ( int ipagV = 0; ipagV < npagV; ++ipagV ) #endif @@ -1913,13 +1920,27 @@ namespace Proc #ifdef __CUDACC__ const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid allMEs[ievt] += deltaMEs; - //printf( "calculate_wavefunction: %6d %2d %f\n", ievt, ihel, allMEs[ievt] ); + //if ( cNGoodHel > 0 ) printf( "calculate_wavefunction: %6d %2d %f\n", ievt, ihel, allMEs[ievt] ); +#else +#ifdef MGONGPU_CPPSIMD + if ( isAligned_allMEs ) + { + *reinterpret_cast( &( allMEs[ipagV*neppV] ) ) += deltaMEs; + } + else + { + for ( int ieppV=0; ieppV 0 ) + // for ( int ieppV=0; ieppV 0 ) printf( "calculate_wavefunction: %6d %2d %f\n", ipagV, ihel, allMEs[ipagV] ); +#endif #endif } - mgDebug( 1, __FUNCTION__ ); return; } @@ -2011,10 +2032,13 @@ namespace Proc memcpy( cHel, tHel, ncomb * mgOnGpu::npar * sizeof(short) ); #endif // SANITY CHECK: GPU memory usage may be based on casts of fptype[2] to cxtype - assert( sizeof(cxtype) == 2 * sizeof(fptype) ); + static_assert( sizeof(cxtype) == 2 * sizeof(fptype) ); #ifndef __CUDACC__ - // SANITY CHECK: momenta AOSOA uses vectors with the same size as fptype_v - assert( neppV == mgOnGpu::neppM ); + // SANITY CHECK: check that neppR, neppM and neppV are powers of two (https://stackoverflow.com/a/108360) + auto ispoweroftwo = []( int n ) { return ( n > 0 ) && !( n & ( n - 1 ) ); }; + static_assert( ispoweroftwo( mgOnGpu::neppR ) ); + static_assert( ispoweroftwo( mgOnGpu::neppM ) ); + static_assert( ispoweroftwo( neppV ) ); #endif } @@ -2126,9 +2150,9 @@ namespace Proc #ifdef __CUDACC__ __global__ - void sigmaKin_getGoodHel( const fptype_sv* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM - fptype_sv* allMEs, // output: allMEs[npagM][neppM], final |M|^2 averaged over helicities - bool* isGoodHel ) // output: isGoodHel[ncomb] - device array + void sigmaKin_getGoodHel( const fptype* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM + fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities + bool* isGoodHel ) // output: isGoodHel[ncomb] - device array { const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid // FIXME: assume process.nprocesses == 1 for the moment (eventually: need a loop over processes here?) @@ -2146,33 +2170,35 @@ namespace Proc } } #else - void sigmaKin_getGoodHel( const fptype_sv* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM - fptype_sv* allMEs, // output: allMEs[npagM][neppM], final |M|^2 averaged over helicities - bool* isGoodHel // output: isGoodHel[ncomb] - device array - , const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + void sigmaKin_getGoodHel( const fptype* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM + fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities + bool* isGoodHel // output: isGoodHel[ncomb] - device array + , const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) { + assert( (size_t)(allmomenta) % mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment + //assert( (size_t)(allMEs) % mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment const int maxtry0 = ( neppV > 16 ? neppV : 16 ); // 16, but at least neppV (otherwise the npagV loop does not even start) - fptype_sv allMEsLast[maxtry0/neppV] = { 0 }; + fptype allMEsLast[maxtry0] = { 0 }; const int maxtry = std::min( maxtry0, nevt ); // 16, but at most nevt (avoid invalid memory access if nevt allMEs[nevt*nprocesses]?) __global__ - void sigmaKin( const fptype_sv* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM - fptype_sv* allMEs // output: allMEs[npagM][neppM], final |M|^2 averaged over helicities + void sigmaKin( const fptype* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM + fptype* allMEs // output: allMEs[nevt], |M|^2 final_avg_over_helicities #ifndef __CUDACC__ - , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) #endif ) { @@ -2242,7 +2268,8 @@ namespace Proc const int npagV = nevt/neppV; for ( int ipagV = 0; ipagV < npagV; ++ipagV ) { - allMEs[ipagV] = fptype_sv{ 0 }; + for ( int ieppV=0; ieppV( nRnarray ); // AOSOA[npagR][nparf][np4][neppR] (NB: nevt=npagR*neppR) + auto hstRnarray = hstMakeUnique( nRnarray ); // AOSOA[npagR][nparf][np4][neppR] (NB: nevt=npagR*neppR) #endif - auto hstMomenta = hstMakeUnique( nMomenta ); // AOSOA[npagM][npar][np4][neppM] (NB: nevt=npagM*neppM) - auto hstIsGoodHel = hstMakeUnique( ncomb ); - auto hstWeights = hstMakeUnique( nWeights ); - auto hstMEs = hstMakeUnique( nMEs ); // AOSOA[npagM][neppM] (NB: nevt=npagM*neppM) + auto hstMomenta = hstMakeUnique( nMomenta ); // AOSOA[npagM][npar][np4][neppM] (NB: nevt=npagM*neppM) + auto hstIsGoodHel = hstMakeUnique( ncomb ); + auto hstWeights = hstMakeUnique( nWeights ); + auto hstMEs = hstMakeUnique( nMEs ); // ARRAY[nevt] #ifdef __CUDACC__ - auto devRnarray = devMakeUnique( nRnarray ); // AOSOA[npagR][nparf][np4][neppR] (NB: nevt=npagR*neppR) - auto devMomenta = devMakeUnique( nMomenta ); // AOSOA[npagM][npar][np4][neppM] (NB: nevt=npagM*neppM) - auto devIsGoodHel = devMakeUnique( ncomb ); - auto devWeights = devMakeUnique( nWeights ); - auto devMEs = devMakeUnique( nMEs ); // AOSOA[npagM][neppM] (NB: nevt=npagM*neppM) + auto devRnarray = devMakeUnique( nRnarray ); // AOSOA[npagR][nparf][np4][neppR] (NB: nevt=npagR*neppR) + auto devMomenta = devMakeUnique( nMomenta ); // AOSOA[npagM][npar][np4][neppM] (NB: nevt=npagM*neppM) + auto devIsGoodHel = devMakeUnique( ncomb ); + auto devWeights = devMakeUnique( nWeights ); + auto devMEs = devMakeUnique( nMEs ); // ARRAY[nevt] #if defined MGONGPU_CURAND_ONHOST or defined MGONGPU_COMMONRAND_ONHOST const int nbytesRnarray = nRnarray * sizeof(fptype); @@ -553,37 +553,22 @@ int main(int argc, char **argv) // NB: 'setw' affects only the next field (of any type) std::cout << std::scientific // fixed format: affects all floats (default precision: 6) << std::setw(4) << ipar + 1 -#ifndef MGONGPU_CPPSIMD << std::setw(14) << hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + 0*neppM + ieppM] // AOSOA[ipagM][ipar][0][ieppM] << std::setw(14) << hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + 1*neppM + ieppM] // AOSOA[ipagM][ipar][1][ieppM] << std::setw(14) << hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + 2*neppM + ieppM] // AOSOA[ipagM][ipar][2][ieppM] << std::setw(14) << hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + 3*neppM + ieppM] // AOSOA[ipagM][ipar][3][ieppM] -#else - << std::setw(14) << hstMomenta[ipagM*npar*np4 + ipar*np4 + 0][ieppM] // AOSOA[ipagM][ipar][0][ieppM] - << std::setw(14) << hstMomenta[ipagM*npar*np4 + ipar*np4 + 1][ieppM] // AOSOA[ipagM][ipar][1][ieppM] - << std::setw(14) << hstMomenta[ipagM*npar*np4 + ipar*np4 + 2][ieppM] // AOSOA[ipagM][ipar][2][ieppM] - << std::setw(14) << hstMomenta[ipagM*npar*np4 + ipar*np4 + 3][ieppM] // AOSOA[ipagM][ipar][3][ieppM] -#endif << std::endl << std::defaultfloat; // default format: affects all floats } std::cout << std::string(SEP79, '-') << std::endl; // Display matrix elements std::cout << " Matrix element = " -#ifndef MGONGPU_CPPSIMD << hstMEs[ievt] -#else - << hstMEs[ievt/neppM][ievt%neppM] -#endif << " GeV^" << meGeVexponent << std::endl; // FIXME: assume process.nprocesses == 1 std::cout << std::string(SEP79, '-') << std::endl; } // Fill the arrays with ALL MEs and weights -#ifndef MGONGPU_CPPSIMD matrixelementALL[iiter*nevt + ievt] = hstMEs[ievt]; // FIXME: assume process.nprocesses == 1 -#else - matrixelementALL[iiter*nevt + ievt] = hstMEs[ievt/neppM][ievt%neppM]; // FIXME: assume process.nprocesses == 1 -#endif weightALL[iiter*nevt + ievt] = hstWeights[ievt]; } diff --git a/epochX/cudacpp/gg_ttgg.auto/SubProcesses/runTest.cc b/epochX/cudacpp/gg_ttgg.auto/SubProcesses/runTest.cc index d92ffe4e7a..a4f3519434 100644 --- a/epochX/cudacpp/gg_ttgg.auto/SubProcesses/runTest.cc +++ b/epochX/cudacpp/gg_ttgg.auto/SubProcesses/runTest.cc @@ -21,7 +21,7 @@ template using unique_ptr_host = std::unique_ptr>; #else template -using unique_ptr_host = std::unique_ptr; +using unique_ptr_host = std::unique_ptr>; #endif struct CUDA_CPU_TestBase : public TestDriverBase { @@ -47,11 +47,11 @@ struct CPUTest : public CUDA_CPU_TestBase { // Struct data members (process, and memory structures for random numbers, momenta, matrix elements and weights on host and device) // [NB the hst/dev memory arrays must be initialised in the constructor, see issue #290] Proc::CPPProcess process; - unique_ptr_host hstRnarray; - unique_ptr_host hstMomenta; - unique_ptr_host hstIsGoodHel; - unique_ptr_host hstWeights; - unique_ptr_host hstMEs; + unique_ptr_host hstRnarray; + unique_ptr_host hstMomenta; + unique_ptr_host hstIsGoodHel; + unique_ptr_host hstWeights; + unique_ptr_host hstMEs; // Create a process object // Read param_card and set parameters @@ -61,11 +61,11 @@ struct CPUTest : public CUDA_CPU_TestBase { CPUTest( const std::string& refFileName ) : CUDA_CPU_TestBase( refFileName ), process(niter, gpublocks, gputhreads, /*verbose=*/false), - hstRnarray { hstMakeUnique( nRnarray ) }, // AOSOA[npagR][nparf][np4][neppR] - hstMomenta { hstMakeUnique( nMomenta ) }, // AOSOA[npagM][npar][np4][neppM] - hstIsGoodHel{ hstMakeUnique( mgOnGpu::ncomb ) }, - hstWeights { hstMakeUnique( nWeights ) }, - hstMEs { hstMakeUnique( nMEs ) } // AOSOA[npagM][neppM] + hstRnarray { hstMakeUnique( nRnarray ) }, // AOSOA[npagR][nparf][np4][neppR] + hstMomenta { hstMakeUnique( nMomenta ) }, // AOSOA[npagM][npar][np4][neppM] + hstIsGoodHel{ hstMakeUnique( mgOnGpu::ncomb ) }, + hstWeights { hstMakeUnique( nWeights ) }, + hstMEs { hstMakeUnique( nMEs ) } // ARRAY[nevt] { process.initProc("../../Cards/param_card.dat"); } @@ -106,19 +106,11 @@ struct CPUTest : public CUDA_CPU_TestBase { assert(particle < npar); const auto ipagM = evtNo / neppM; // #eventpage in this iteration const auto ieppM = evtNo % neppM; // #event in the current eventpage in this iteration -#ifndef MGONGPU_CPPSIMD return hstMomenta[ipagM*npar*np4*neppM + particle*np4*neppM + component*neppM + ieppM]; -#else - return hstMomenta[ipagM*npar*np4 + particle*np4 + component][ieppM]; -#endif }; fptype getMatrixElement(std::size_t ievt) const override { -#ifndef MGONGPU_CPPSIMD return hstMEs[ievt]; -#else - return hstMEs[ievt/neppV][ievt%neppV]; -#endif } }; @@ -161,12 +153,12 @@ struct CUDATest : public CUDA_CPU_TestBase { hstMomenta { hstMakeUnique( nMomenta ) }, // AOSOA[npagM][npar][np4][neppM] (nevt=npagM*neppM) hstIsGoodHel{ hstMakeUnique( mgOnGpu::ncomb ) }, hstWeights { hstMakeUnique( nWeights ) }, - hstMEs { hstMakeUnique( nMEs ) }, + hstMEs { hstMakeUnique( nMEs ) }, // ARRAY[nevt] devRnarray { devMakeUnique( nRnarray ) }, // AOSOA[npagR][nparf][np4][neppR] (nevt=npagR*neppR) - devMomenta { devMakeUnique( nMomenta ) }, + devMomenta { devMakeUnique( nMomenta ) }, // AOSOA[npagM][npar][np4][neppM] (nevt=npagM*neppM) devIsGoodHel{ devMakeUnique( mgOnGpu::ncomb ) }, devWeights { devMakeUnique( nWeights ) }, - devMEs { devMakeUnique( nMEs ) } + devMEs { devMakeUnique( nMEs ) } // ARRAY[nevt] { process.initProc("../../Cards/param_card.dat"); } diff --git a/epochX/cudacpp/gg_ttgg.auto/SubProcesses/testxxx.cc b/epochX/cudacpp/gg_ttgg.auto/SubProcesses/testxxx.cc index 6df896c75f..9ae06a7404 100644 --- a/epochX/cudacpp/gg_ttgg.auto/SubProcesses/testxxx.cc +++ b/epochX/cudacpp/gg_ttgg.auto/SubProcesses/testxxx.cc @@ -31,8 +31,8 @@ TEST( XTESTID_CPU( MG_EPOCH_PROCESS_ID ), testxxx ) assert( nevt % neppM == 0 ); // nevt must be a multiple of neppM // Fill in the input momenta const int nMomenta = np4 * npar * nevt; - auto hstMomenta = hstMakeUnique( nMomenta ); // AOSOA[npagM][npar=4][np4=4][neppM] - const fptype par0[np4 * nevt] = // AOS[nevt][np4] + auto hstMomenta = hstMakeUnique( nMomenta ); // AOSOA[npagM][npar=4][np4=4][neppM] + const fptype par0[np4 * nevt] = // AOS[nevt][np4] { 500, 0, 0, 500, // #0 (m=0 pT=0 E=pz>0) 500, 0, 0, -500, // #1 (m=0 pT=0 -E=pz<0) 500, 300, 400, 0, // #2 (m=0 pT>0 pz=0) @@ -72,11 +72,7 @@ TEST( XTESTID_CPU( MG_EPOCH_PROCESS_ID ), testxxx ) { const int ipagM = ievt/neppM; // #eventpage in this iteration const int ieppM = ievt%neppM; // #event in the current eventpage in this iteration -#ifdef MGONGPU_CPPSIMD - hstMomenta[ipagM*npar*np4 + ipar*np4 + ip4][ieppM] = par0[ievt*np4 + ip4]; // AOS to AOSOA -#else hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + ip4*neppM + ieppM] = par0[ievt*np4 + ip4]; // AOS to AOSOA -#endif } } // Expected output wavefunctions diff --git a/epochX/cudacpp/gg_ttgg.auto/src/HelAmps_sm.cc b/epochX/cudacpp/gg_ttgg.auto/src/HelAmps_sm.cc index 62dbaa3768..b70f901581 100644 --- a/epochX/cudacpp/gg_ttgg.auto/src/HelAmps_sm.cc +++ b/epochX/cudacpp/gg_ttgg.auto/src/HelAmps_sm.cc @@ -23,8 +23,8 @@ namespace MG5_sm //-------------------------------------------------------------------------- -#ifdef __CUDACC__ - // Return by reference + // Decode momentum AOSOA: compute address of fptype for the given particle, 4-momentum component and event + // Return the fptype by reference (equivalent to returning its memory address) __device__ inline const fptype& pIparIp4Ievt( const fptype* momenta, // input: momenta as AOSOA[npagM][npar][4][neppM] const int ipar, @@ -33,38 +33,149 @@ namespace MG5_sm { using mgOnGpu::np4; using mgOnGpu::npar; - const int neppM = mgOnGpu::neppM; // AOSOA layout: constant at compile-time - const int ipagM = ievt/neppM; // #eventpage in this iteration - const int ieppM = ievt%neppM; // #event in the current eventpage in this iteration - //printf( "%f\n", momenta[ipagM*npar*np4*neppM + ipar*np4*neppM + ip4*neppM + ieppM] ); + constexpr int neppM = mgOnGpu::neppM; // AOSOA layout: constant at compile-time + const int ipagM = ievt/neppM; // #event "M-page" + const int ieppM = ievt%neppM; // #event in the current event M-page + //printf( "%2d %2d %8d %8.3f\n", ipar, ip4, ievt, momenta[ipagM*npar*np4*neppM + ipar*np4*neppM + ip4*neppM + ieppM] ); return momenta[ipagM*npar*np4*neppM + ipar*np4*neppM + ip4*neppM + ieppM]; // AOSOA[ipagM][ipar][ip4][ieppM] //fptype (*momenta)[npar][np4][neppM] = (fptype (*)[npar][np4][neppM]) momenta; // cast to multiD array pointer (AOSOA) //return momenta[ipagM][ipar][ip4][ieppM]; // this seems ~1-2% faster in eemumu C++? } -#else - // Return by value: it seems a tiny bit faster than returning a reference (both for scalar and vector), not clear why + +#ifndef __CUDACC__ + // Return a SIMD vector of fptype's for neppV events (for the given particle, 4-momentum component and event "V-page") + // Return the vector by value: strictly speaking, this is only unavoidable for neppM=neppV (both being powers of 2), the momentum neppM-AOSOA is reinterpreted in terms of neppV-vectors: + // it could also be returned by reference, but no performance degradation is observed when returning by value inline - fptype_sv pIparIp4Ipag( const fptype_sv* momenta, // input: momenta as AOSOA[npagM][npar][4][neppM] + fptype_sv pIparIp4Ipag( const fptype* momenta, // input: momenta as AOSOA[npagM][npar][4][neppM] const int ipar, const int ip4, - const int ipagM ) + const int ipagV ) { - /* - //#ifndef MGONGPU_CPPSIMD - // TEMPORARY during epochX step3! Eventually remove this section (start) - // TEMPORARY during epochX step3! THERE IS NO SIMD YET: HENCE ipagM=ievt - // NB: this is needed for neppM>1 while neppV==1 (no SIMD) in eemumu.auto - // NB: this remains valid for neppM==1 with neppV==1 in eemumu (scalar) - return pIparIp4Ievt( momenta, ipar, ip4, ipagM ); - // TEMPORARY during epochX step3! Eventually remove this section (end) - //#else - */ - // NB: this assumes that neppV == neppM! - // NB: this is the same as "pIparIp4Ievt( momenta, ipar, ip4, ipagM )" for neppM==1 with neppV==1 - using mgOnGpu::np4; - using mgOnGpu::npar; - //printf( "%f\n", momenta[ipagM*npar*np4 + ipar*np4 + ip4] ); - return momenta[ipagM*npar*np4 + ipar*np4 + ip4]; // AOSOA[ipagM][ipar][ip4][ieppM] + const int ievt0 = ipagV*neppV; // virtual event V-page ipagV contains neppV events [ievt0...ievt0+neppV-1] +#ifdef MGONGPU_CPPSIMD + constexpr int neppM = mgOnGpu::neppM; // AOSOA layout: constant at compile-time + // Use c++17 "if constexpr": compile-time branching + if constexpr ( ( neppM >= neppV ) && ( neppM%neppV == 0 ) ) + { + constexpr bool useReinterpretCastIfPossible = true; // DEFAULT + //constexpr bool useReinterpretCastIfPossible = false; // FOR PERFORMANCE TESTS + constexpr bool skipAlignmentCheck = true; // DEFAULT (MAY SEGFAULT, NEEDS A SANITY CHECK ELSEWHERE!) + //constexpr bool skipAlignmentCheck = false; // SLOWER BUT SAFER + if constexpr ( useReinterpretCastIfPossible && skipAlignmentCheck ) + { + //static bool first=true; if( first ){ std::cout << "WARNING! skip alignment check" << std::endl; first=false; } // SLOWS DOWN... + // Fastest (4.92E6 in eemumu 512y) + // This requires alignment for momenta - segmentation fault otherwise! + return *reinterpret_cast( &( pIparIp4Ievt( momenta, ipar, ip4, ievt0 ) ) ); + } + else if ( useReinterpretCastIfPossible && ( (size_t)(momenta) % mgOnGpu::cppAlign == 0 ) ) + { + //static bool first=true; if( first ){ std::cout << "WARNING! alignment ok, use reinterpret cast" << std::endl; first=false; } // SLOWS DOWN... + // A bit (6%) slower (4.62E6 in eemumu 512y) because of the alignment check + // This explicitly checks alignment for momenta to avoid segmentation faults + return *reinterpret_cast( &( pIparIp4Ievt( momenta, ipar, ip4, ievt0 ) ) ); + } + else + { + //static bool first=true; if( first ){ std::cout << "WARNING! AOSOA but no reinterpret cast" << std::endl; first=false; } // SLOWS DOWN... + // A bit (2%) slower (4.86E6 in eemumu 512y) + // This does not require alignment for momenta, but it requires AOSOA with neppM>=neppV and neppM%neppV==0 + const fptype* out0 = &( pIparIp4Ievt( momenta, ipar, ip4, ievt0) ); +#if MGONGPU_CPPSIMD == 2 + return fptype_v{ *( out0 ), + *( out0+1 ) }; +#elif MGONGPU_CPPSIMD == 4 + return fptype_v{ *( out0 ), + *( out0+1 ), + *( out0+2 ), + *( out0+3 ) }; +#elif MGONGPU_CPPSIMD == 8 + return fptype_v{ *( out0 ), + *( out0+1 ), + *( out0+2 ), + *( out0+3 ), + *( out0+4 ), + *( out0+5 ), + *( out0+6 ), + *( out0+7 ) }; +#elif MGONGPU_CPPSIMD == 16 + return fptype_v{ *( out0 ), + *( out0+1 ), + *( out0+2 ), + *( out0+3 ), + *( out0+4 ), + *( out0+5 ), + *( out0+6 ), + *( out0+7 ), + *( out0+8 ), + *( out0+9 ), + *( out0+10 ), + *( out0+11 ), + *( out0+12 ), + *( out0+13 ), + *( out0+14 ), + *( out0+15 ) }; +#else +#warning Internal error? This code should never be reached! + // A bit (2%) slower (4.86E6 in eemumu 512y) + // This does not require alignment for momenta, but it requires AOSOA with neppM>=neppV and neppM%neppV==0 + fptype_v out; + for ( int ieppV=0; ieppV=neppV and neppM%neppV==0 (e.g. can be used with AOS neppM==1) +#if MGONGPU_CPPSIMD == 2 + return fptype_v{ pIparIp4Ievt( momenta, ipar, ip4, ievt0 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+1 ) }; +#elif MGONGPU_CPPSIMD == 4 + return fptype_v{ pIparIp4Ievt( momenta, ipar, ip4, ievt0 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+1 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+2 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+3 ) }; +#elif MGONGPU_CPPSIMD == 8 + return fptype_v{ pIparIp4Ievt( momenta, ipar, ip4, ievt0 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+1 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+2 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+3 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+4 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+5 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+6 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+7 ) }; +#elif MGONGPU_CPPSIMD == 16 + return fptype_v{ pIparIp4Ievt( momenta, ipar, ip4, ievt0 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+1 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+2 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+3 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+4 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+5 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+6 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+7 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+8 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+9 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+10 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+11 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+12 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+13 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+14 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+15 ) }; +#else +#warning Internal error? This code should never be reached! + // Much much (40%) slower (3.02E6 in eemumu 512y) + fptype_v out; + for ( int ieppV=0; ieppV 0) __device__ - void ipzxxx( const fptype_sv* momenta, + void ipzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -257,7 +368,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) __device__ - void imzxxx( const fptype_sv* momenta, + void imzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -306,7 +417,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PT > 0) __device__ - void ixzxxx( const fptype_sv* momenta, + void ixzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -367,7 +478,7 @@ namespace MG5_sm // Compute the output wavefunction vc[6] from the input momenta[npar*4*nevt] __device__ - void vxxxxx( const fptype_sv* momenta, + void vxxxxx( const fptype* momenta, const fptype vmass, // input: vector boson mass const int nhel, // input: -1, 0 (only if vmass!=0) or +1 (helicity of vector boson) const int nsv, // input: +1 (final) or -1 (initial) @@ -503,7 +614,7 @@ namespace MG5_sm // Compute the output wavefunction sc[3] from the input momenta[npar*4*nevt] __device__ - void sxxxxx( const fptype_sv* momenta, + void sxxxxx( const fptype* momenta, const fptype, // WARNING: input "smass" unused (missing in Fortran) - scalar boson mass const int, // WARNING: input "nhel" unused (missing in Fortran) - scalar has no helicity! const int nss, // input: +1 (final) or -1 (initial) @@ -544,7 +655,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] __device__ - void oxxxxx( const fptype_sv* momenta, + void oxxxxx( const fptype* momenta, const fptype fmass, // input: fermion mass const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -681,7 +792,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) __device__ - void opzxxx( const fptype_sv* momenta, + void opzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -730,7 +841,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) __device__ - void omzxxx( const fptype_sv* momenta, + void omzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -782,7 +893,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PT > 0) __device__ - void oxzxxx( const fptype_sv* momenta, + void oxzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) diff --git a/epochX/cudacpp/gg_ttgg.auto/src/HelAmps_sm.h b/epochX/cudacpp/gg_ttgg.auto/src/HelAmps_sm.h index 087a6d4cdd..529f37c33d 100644 --- a/epochX/cudacpp/gg_ttgg.auto/src/HelAmps_sm.h +++ b/epochX/cudacpp/gg_ttgg.auto/src/HelAmps_sm.h @@ -28,7 +28,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] __device__ INLINE - void ixxxxx( const fptype_sv* momenta, + void ixxxxx( const fptype* momenta, const fptype fmass, // input: fermion mass const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -43,7 +43,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) __device__ INLINE - void ipzxxx( const fptype_sv* momenta, + void ipzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -58,7 +58,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) __device__ INLINE - void imzxxx( const fptype_sv* momenta, + void imzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -73,7 +73,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PT > 0) __device__ INLINE - void ixzxxx( const fptype_sv* momenta, + void ixzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -87,7 +87,7 @@ namespace MG5_sm // Compute the output wavefunction vc[6] from the input momenta[npar*4*nevt] __device__ INLINE - void vxxxxx( const fptype_sv* momenta, + void vxxxxx( const fptype* momenta, const fptype vmass, // input: vector boson mass const int nhel, // input: -1, 0 (only if vmass!=0) or +1 (helicity of vector boson) const int nsv, // input: +1 (final) or -1 (initial) @@ -101,7 +101,7 @@ namespace MG5_sm // Compute the output wavefunction sc[3] from the input momenta[npar*4*nevt] __device__ INLINE - void sxxxxx( const fptype_sv* momenta, + void sxxxxx( const fptype* momenta, const fptype, // WARNING: input "smass" unused (missing in Fortran) - scalar boson mass const int, // WARNING: input "nhel" unused (missing in Fortran) - scalar has no helicity! const int nss, // input: +1 (final) or -1 (initial) @@ -115,7 +115,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] __device__ INLINE - void oxxxxx( const fptype_sv* momenta, + void oxxxxx( const fptype* momenta, const fptype fmass, // input: fermion mass const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -130,7 +130,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) __device__ INLINE - void opzxxx( const fptype_sv* momenta, + void opzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -145,7 +145,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) __device__ INLINE - void omzxxx( const fptype_sv* momenta, + void omzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -159,7 +159,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] __device__ INLINE - void oxzxxx( const fptype_sv* momenta, + void oxzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) diff --git a/epochX/cudacpp/gg_ttgg.auto/src/Makefile b/epochX/cudacpp/gg_ttgg.auto/src/Makefile index ce74470562..6e6b6a0c39 100644 --- a/epochX/cudacpp/gg_ttgg.auto/src/Makefile +++ b/epochX/cudacpp/gg_ttgg.auto/src/Makefile @@ -160,7 +160,7 @@ all.$(TAG): $(BUILDDIR)/.build.$(TAG) $(target) override oldtags=`find $(BUILDDIR) -maxdepth 1 -name '.build.*' ! -name '.build.$(TAG)'` $(BUILDDIR)/.build.$(TAG): - @if [ ! -d $(BUILDDIR) ]; then echo "mkdir $(BUILDDIR)"; mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then echo "mkdir -p $(BUILDDIR)"; mkdir -p $(BUILDDIR); fi @if [ "$(oldtags)" != "" ]; then echo -e "Cannot build for tag=$(TAG) as old builds exist for other tags:\n$(oldtags)\nPlease run 'make clean' first\nIf 'make clean' is not enough: run 'make clean USEBUILDDIR=1 AVX=$(AVX) FPTYPE=$(FPTYPE)' or 'make cleanall'"; exit 1; fi @touch $(BUILDDIR)/.build.$(TAG) @@ -171,11 +171,11 @@ debug: $(target) # NB: cuda includes are needed in the C++ code for curand.h $(BUILDDIR)/%.o : %.cc *.h - @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(CUINC) -c $< -o $@ #$(BUILDDIR)/%.o : %.cu *.h -# @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi +# @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi # $(NVCC) $(CPPFLAGS) $(CUFLAGS) -c $< -o $@ $(target): $(cxx_objects) $(cu_objects) diff --git a/epochX/cudacpp/gg_ttgg.auto/src/mgOnGpuConfig.h b/epochX/cudacpp/gg_ttgg.auto/src/mgOnGpuConfig.h index ab498c8f10..c2b6f6bacb 100644 --- a/epochX/cudacpp/gg_ttgg.auto/src/mgOnGpuConfig.h +++ b/epochX/cudacpp/gg_ttgg.auto/src/mgOnGpuConfig.h @@ -96,47 +96,79 @@ namespace mgOnGpu //const int ntpbMAX = 256; // AV Apr2021: why had I set this to 256? const int ntpbMAX = 1024; // NB: 512 is ok, but 1024 does fail with "too many resources requested for launch" - // Vector sizes for AOSOA memory layouts (GPU coalesced memory access, CPU SIMD vectorization) - // (these are all best kept as a compile-time constants: see issue #23) + // Alignment requirement for using reinterpret_cast with SIMD vectorized code + // (using reinterpret_cast with non aligned memory may lead to segmentation faults!) +#ifndef __CUDACC__ + constexpr int cppAlign = 64; // alignment requirement for SIMD vectorization (64-byte i.e. 512-bit) +#endif - // Number of Events Per Page in the momenta AOSOA memory layout + // C++ SIMD vectorization width (this will be used to set neppV) #ifdef __CUDACC__ #undef MGONGPU_CPPSIMD - // ----------------------------------------------------------------------------------------- - // --- GPUs: neppM must be a power of 2 times the number of fptype's in a 32-byte cacheline - // --- This is relevant to ensure coalesced access to momenta in global memory - // --- Note that neppR is hardcoded and may differ from neppM and neppV on some platforms - // ----------------------------------------------------------------------------------------- - //const int neppM = 64/sizeof(fptype); // 2x 32-byte GPU cache lines: 8 (DOUBLE) or 16 (FLOAT) - const int neppM = 32/sizeof(fptype); // (DEFAULT) 32-byte GPU cache line: 4 (DOUBLE) or 8 (FLOAT) - //const int neppM = 1; // *** NB: this is equivalent to AOS *** - //const int neppM = 32; // older default #else - // ----------------------------------------------------------------------------------------- - // --- CPUs: neppM must be exactly equal to the number of fptype's in a vector register - // --- [DEFAULT is 256-width AVX512 aka "512y" on CPUs, 32-byte as GPUs, faster than AVX2] - // --- The logic of the code requires the size neppV of fptype_v to be equal to neppM - // --- Note that neppR is hardcoded and may differ from neppM and neppV on some platforms - // ----------------------------------------------------------------------------------------- #if defined __AVX512VL__ -#define MGONGPU_CPPSIMD 1 #ifdef MGONGPU_PVW512 - const int neppM = 64/sizeof(fptype); // "512z" AVX512 with 512 width (512-bit ie 64-byte): 8 (DOUBLE) or 16 (FLOAT) + // "512z" AVX512 with 512 width (512-bit ie 64-byte): 8 (DOUBLE) or 16 (FLOAT) +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 8 #else - const int neppM = 32/sizeof(fptype); // "512y" AVX512 with 256 width (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [gcc DEFAULT] +#define MGONGPU_CPPSIMD 16 +#endif +#else + // "512y" AVX512 with 256 width (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [gcc DEFAULT] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 4 +#else +#define MGONGPU_CPPSIMD 8 +#endif #endif #elif defined __AVX2__ -#define MGONGPU_CPPSIMD 1 - const int neppM = 32/sizeof(fptype); // "avx2" AVX2 (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [clang DEFAULT] + // "avx2" AVX2 (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [clang DEFAULT] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 4 +#else +#define MGONGPU_CPPSIMD 8 +#endif #elif defined __SSE4_2__ -#define MGONGPU_CPPSIMD 1 - const int neppM = 16/sizeof(fptype); // "sse4" SSE4.2 (128-bit ie 16-byte): 2 (DOUBLE) or 4 (FLOAT) + // "sse4" SSE4.2 (128-bit ie 16-byte): 2 (DOUBLE) or 4 (FLOAT) +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 2 #else +#define MGONGPU_CPPSIMD 4 +#endif +#else + // "none" i.e. no SIMD (*** NB: this is equivalent to AOS ***) #undef MGONGPU_CPPSIMD - const int neppM = 1; // "none" i.e. no SIMD (*** NB: this is equivalent to AOS ***) #endif #endif + // Number of Events Per Page in the momenta AOSOA memory layout +#ifdef __CUDACC__ + // ----------------------------------------------------------------------------------------------- + // --- GPUs: neppM is best set to a power of 2 times the number of fptype's in a 32-byte cacheline + // --- This is relevant to ensure coalesced access to momenta in global memory + // --- Note that neppR is hardcoded and may differ from neppM and neppV on some platforms + // ----------------------------------------------------------------------------------------------- + //const int neppM = 64/sizeof(fptype); // 2x 32-byte GPU cache lines (512 bits): 8 (DOUBLE) or 16 (FLOAT) + const int neppM = 32/sizeof(fptype); // (DEFAULT) 32-byte GPU cache line (256 bits): 4 (DOUBLE) or 8 (FLOAT) + //const int neppM = 1; // *** NB: this is equivalent to AOS *** +#else + // ----------------------------------------------------------------------------------------------- + // --- CPUs: neppM is best set equal to the number of fptype's (neppV) in a vector register + // --- This is relevant to ensure faster access to momenta from C++ memory cache lines + // --- However, neppM is now decoupled from neppV (issue #176) and can be separately hardcoded + // --- In practice, neppR, neppM and neppV could now (in principle) all be different + // ----------------------------------------------------------------------------------------------- +#ifdef MGONGPU_CPPSIMD + const int neppM = MGONGPU_CPPSIMD; // (DEFAULT) neppM=neppV for optimal performance +#else + const int neppM = 1; // (DEFAULT) neppM=neppV for optimal performance (NB: this is equivalent to AOS) +#endif + //const int neppM = 64/sizeof(fptype); // maximum CPU vector width (512 bits): 8 (DOUBLE) or 16 (FLOAT) + //const int neppM = 32/sizeof(fptype); // lower CPU vector width (256 bits): 4 (DOUBLE) or 8 (FLOAT) + //const int neppM = 1; // *** NB: this is equivalent to AOS *** +#endif + // Number of Events Per Page in the random number AOSOA memory layout // *** NB Different values of neppR lead to different physics results: the *** // *** same 1d array is generated, but it is interpreted in different ways *** diff --git a/epochX/cudacpp/gg_ttgg.auto/src/mgOnGpuVectors.h b/epochX/cudacpp/gg_ttgg.auto/src/mgOnGpuVectors.h index e1da365663..4e975b2ca1 100644 --- a/epochX/cudacpp/gg_ttgg.auto/src/mgOnGpuVectors.h +++ b/epochX/cudacpp/gg_ttgg.auto/src/mgOnGpuVectors.h @@ -13,9 +13,13 @@ namespace mgOnGpu { + #ifdef MGONGPU_CPPSIMD - const int neppV = neppM; + const int neppV = MGONGPU_CPPSIMD; + + // SANITY CHECK: cppAlign must be a multiple of neppV * sizeof(fptype) + static_assert( mgOnGpu::cppAlign % ( neppV * sizeof(fptype) ) == 0 ); // --- Type definition (using vector compiler extensions: need -march=...) #ifdef __clang__ // https://clang.llvm.org/docs/LanguageExtensions.html#vectors-and-extended-vectors @@ -96,7 +100,7 @@ namespace mgOnGpu #else // MGONGPU_CPPSIMD not defined - const int neppV = 1; // Note: also neppM is equal to 1 + const int neppV = 1; #endif } @@ -114,7 +118,7 @@ using mgOnGpu::bool_v; inline std::ostream& operator<<( std::ostream& out, const bool_v& v ) { out << "{ " << v[0]; - for ( int i=1; i #include @@ -22,7 +22,7 @@ namespace rambo2toNm0 // [NB: the output buffer includes both initial and final momenta, but only initial momenta are filled in] __global__ void getMomentaInitial( const fptype energy, // input: energy - fptype_sv momenta1d[] // output: momenta as AOSOA[npagM][npar][4][neppM] + fptype momenta1d[] // output: momenta as AOSOA[npagM][npar][4][neppM] #ifndef __CUDACC__ , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) #endif @@ -63,7 +63,7 @@ namespace rambo2toNm0 __global__ void getMomentaFinal( const fptype energy, // input: energy const fptype rnarray1d[], // input: random numbers in [0,1] as AOSOA[npagR][nparf][4][neppR] - fptype_sv momenta1d[], // output: momenta as AOSOA[npagM][npar][4][neppM] + fptype momenta1d[], // output: momenta as AOSOA[npagM][npar][4][neppM] fptype wgts[] // output: weights[nevt] #ifndef __CUDACC__ , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) diff --git a/epochX/cudacpp/gg_ttgg.auto/src/rambo.h b/epochX/cudacpp/gg_ttgg.auto/src/rambo.h index 030526759f..9d9e876fe3 100644 --- a/epochX/cudacpp/gg_ttgg.auto/src/rambo.h +++ b/epochX/cudacpp/gg_ttgg.auto/src/rambo.h @@ -3,7 +3,7 @@ #include "mgOnGpuConfig.h" #include "mgOnGpuTypes.h" -#include "mgOnGpuVectors.h" +//#include "mgOnGpuVectors.h" #include @@ -47,7 +47,7 @@ namespace rambo2toNm0 // [NB: the output buffer includes both initial and final momenta, but only initial momenta are filled in] __global__ void getMomentaInitial( const fptype energy, // input: energy - fptype_sv momenta1d[] // output: momenta as AOSOA[npagM][npar][4][neppM] + fptype momenta1d[] // output: momenta as AOSOA[npagM][npar][4][neppM] #ifndef __CUDACC__ , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) #endif @@ -60,7 +60,7 @@ namespace rambo2toNm0 __global__ void getMomentaFinal( const fptype energy, // input: energy const fptype rnarray1d[], // input: random numbers in [0,1] as AOSOA[npagR][nparf][4][neppR] - fptype_sv momenta1d[], // output: momenta as AOSOA[npagM][npar][4][neppM] + fptype momenta1d[], // output: momenta as AOSOA[npagM][npar][4][neppM] fptype wgts[] // output: weights[nevt] #ifndef __CUDACC__ , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) From 65f7b2d376af8e0947877365c6474b4f3da4c799 Mon Sep 17 00:00:00 2001 From: Andrea Valassi Date: Wed, 1 Dec 2021 11:39:49 +0100 Subject: [PATCH 21/34] [klas5X] rerun all ggttgg auto tests, all ok - essentially the same performance --- .../log_ggttgg_auto_d_inl0.txt | 94 +++++++------- .../log_ggttgg_auto_d_inl1.txt | 120 +++++++++--------- .../log_ggttgg_auto_f_inl0.txt | 94 +++++++------- .../log_ggttgg_auto_f_inl1.txt | 120 +++++++++--------- 4 files changed, 214 insertions(+), 214 deletions(-) diff --git a/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_d_inl0.txt b/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_d_inl0.txt index 4d4b48d3ae..e1e3466467 100644 --- a/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_d_inl0.txt +++ b/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_d_inl0.txt @@ -40,45 +40,45 @@ RNDGEN=curdev Building in BUILDDIR=build.512z_d_inl0 for tag=512z_d_inl0_curdev (USEBUILDDIR is set = 1) make: Nothing to be done for `all.512z_d_inl0_curdev'. -DATE: 2021-11-16_14:29:05 +DATE: 2021-12-01_11:33:25 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl0/gcheck.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 4.396384e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.399092e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.388543e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.391508e+05 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 0.520641 sec - 196,191,166 cycles:u # 0.730 GHz - 289,602,789 instructions:u # 1.48 insn per cycle - 0.585142234 seconds time elapsed +TOTAL : 5.406831 sec + 12,424,319,754 cycles:u # 2.322 GHz + 23,282,976,350 instructions:u # 1.87 insn per cycle + 5.697459970 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 255 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl0/gcheck.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 4.886185e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.887767e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.886991e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.888706e+05 ) sec^-1 MeanMatrixElemValue = ( 6.665112e+00 +- 5.002651e+00 ) GeV^-4 -TOTAL : 2.336771 sec - 2,231,131,789 cycles:u # 0.932 GHz - 4,387,107,560 instructions:u # 1.97 insn per cycle - 2.400015717 seconds time elapsed +TOTAL : 2.717279 sec + 2,120,203,860 cycles:u # 0.718 GHz + 4,508,397,222 instructions:u # 2.13 insn per cycle + 3.016722705 seconds time elapsed ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl0/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.831970e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.831970e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.829510e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.829510e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 8.978953 sec - 23,916,279,763 cycles:u # 2.664 GHz - 74,429,675,900 instructions:u # 3.11 insn per cycle - 8.985704953 seconds time elapsed +TOTAL : 8.985019 sec + 23,984,698,394 cycles:u # 2.669 GHz + 74,429,676,608 instructions:u # 3.10 insn per cycle + 8.991662009 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 1470) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl0/runTest.exe @@ -88,14 +88,14 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProce Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[2] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 3.356492e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.356492e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 3.376446e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.376446e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 4.908570 sec - 13,071,725,383 cycles:u # 2.662 GHz - 39,630,355,364 instructions:u # 3.03 insn per cycle - 4.915579750 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 9242) (avx2: 0) (512y: 0) (512z: 0) +TOTAL : 4.877238 sec + 13,014,333,492 cycles:u # 2.667 GHz + 39,610,810,845 instructions:u # 3.04 insn per cycle + 4.883926335 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 9180) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_d_inl0/runTest.exe [ PASSED ] 3 tests. @@ -104,14 +104,14 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProce Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 6.866639e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 6.866639e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 6.885384e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 6.885384e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 2.408323 sec - 5,461,315,423 cycles:u # 2.265 GHz - 13,651,281,160 instructions:u # 2.50 insn per cycle - 2.415183164 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 7704) (512y: 0) (512z: 0) +TOTAL : 2.402076 sec + 5,451,716,257 cycles:u # 2.266 GHz + 13,654,534,285 instructions:u # 2.50 insn per cycle + 2.408731848 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 7734) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_d_inl0/runTest.exe [ PASSED ] 3 tests. @@ -120,14 +120,14 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProce Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 7.675746e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 7.675746e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 7.684325e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 7.684325e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 2.156721 sec - 4,889,782,235 cycles:u # 2.264 GHz - 12,417,084,636 instructions:u # 2.54 insn per cycle - 2.163430615 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 7358) (512y: 61) (512z: 0) +TOTAL : 2.153827 sec + 4,888,081,054 cycles:u # 2.265 GHz + 12,420,337,101 instructions:u # 2.54 insn per cycle + 2.161066835 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 7385) (512y: 54) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_d_inl0/runTest.exe [ PASSED ] 3 tests. @@ -136,14 +136,14 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProce Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 6.513098e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 6.513098e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 6.512901e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 6.512901e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 2.538507 sec - 4,003,290,959 cycles:u # 1.575 GHz - 6,319,714,276 instructions:u # 1.58 insn per cycle - 2.545271988 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1836) (512y: 74) (512z: 6010) +TOTAL : 2.538882 sec + 4,004,155,725 cycles:u # 1.575 GHz + 6,320,992,069 instructions:u # 1.58 insn per cycle + 2.545872732 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1904) (512y: 72) (512z: 5995) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_d_inl0/runTest.exe [ PASSED ] 3 tests. diff --git a/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_d_inl1.txt b/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_d_inl1.txt index 5c09240986..cde118c374 100644 --- a/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_d_inl1.txt +++ b/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_d_inl1.txt @@ -1,5 +1,5 @@ -/afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg +/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg OMPFLAGS= AVX=none FPTYPE=d @@ -40,112 +40,112 @@ RNDGEN=curdev Building in BUILDDIR=build.512z_d_inl1 for tag=512z_d_inl1_curdev (USEBUILDDIR is set = 1) make: Nothing to be done for `all.512z_d_inl1_curdev'. -DATE: 2021-10-30_18:33:23 +DATE: 2021-12-01_11:34:08 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl1/gcheck.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl1/gcheck.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 2.759865e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 2.761403e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 2.743772e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 2.745221e+05 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 8.810741 sec - 20,498,814,252 cycles:u # 2.306 GHz - 39,312,269,575 instructions:u # 1.92 insn per cycle - 9.110551643 seconds time elapsed +TOTAL : 8.800843 sec + 20,514,940,846 cycles:u # 2.334 GHz + 39,330,070,250 instructions:u # 1.92 insn per cycle + 9.099733830 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 255 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl1/gcheck.exe -p 2048 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl1/gcheck.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 3.235137e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.235859e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 3.233892e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.234663e+05 ) sec^-1 MeanMatrixElemValue = ( 6.665112e+00 +- 5.002651e+00 ) GeV^-4 -TOTAL : 3.812562 sec - 3,204,986,045 cycles:u # 0.788 GHz - 6,495,315,451 instructions:u # 2.03 insn per cycle - 4.129323118 seconds time elapsed +TOTAL : 3.802337 sec + 3,252,305,204 cycles:u # 0.802 GHz + 6,391,528,355 instructions:u # 1.97 insn per cycle + 4.114623733 seconds time elapsed ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl1/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl1/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.670840e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.670840e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.675128e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.675128e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 9.837905 sec - 26,230,415,720 cycles:u # 2.665 GHz - 64,879,355,634 instructions:u # 2.47 insn per cycle - 9.845568668 seconds time elapsed +TOTAL : 9.811676 sec + 26,193,775,743 cycles:u # 2.669 GHz + 64,879,362,711 instructions:u # 2.48 insn per cycle + 9.818873852 seconds time elapsed =Symbols in CPPProcess.o= (~sse4:13340) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_d_inl1/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_d_inl1/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[2] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 2.669194e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 2.669194e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 2.660372e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 2.660372e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 6.170961 sec - 16,444,547,696 cycles:u # 2.665 GHz - 39,259,113,269 instructions:u # 2.39 insn per cycle - 6.177884594 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4:68136) (avx2: 0) (512y: 0) (512z: 0) +TOTAL : 6.184729 sec + 16,501,089,303 cycles:u # 2.667 GHz + 39,698,251,771 instructions:u # 2.41 insn per cycle + 6.191328560 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4:68925) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_d_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_d_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_d_inl1/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_d_inl1/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 6.149470e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 6.149470e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 6.168104e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 6.168104e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 2.687638 sec - 6,097,620,613 cycles:u # 2.266 GHz - 13,071,882,358 instructions:u # 2.14 insn per cycle - 2.694649751 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2:45381) (512y: 0) (512z: 0) +TOTAL : 2.680501 sec + 6,078,293,584 cycles:u # 2.265 GHz + 13,044,180,861 instructions:u # 2.15 insn per cycle + 2.687213411 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2:45306) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_d_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_d_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_d_inl1/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_d_inl1/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 5.663119e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 5.663119e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 5.677635e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 5.677635e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 2.915918 sec - 6,617,502,580 cycles:u # 2.266 GHz - 12,168,442,885 instructions:u # 1.84 insn per cycle - 2.922933773 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2:41747) (512y: 222) (512z: 0) +TOTAL : 2.908723 sec + 6,601,374,610 cycles:u # 2.267 GHz + 12,188,498,566 instructions:u # 1.85 insn per cycle + 2.915453886 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2:41857) (512y: 215) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_d_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_d_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_d_inl1/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_d_inl1/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 7.242604e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 7.242604e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 7.254946e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 7.254946e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 2.285556 sec - 3,598,507,830 cycles:u # 1.572 GHz - 5,965,707,680 instructions:u # 1.66 insn per cycle - 2.292362801 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1920) (512y: 310) (512z:38736) +TOTAL : 2.282559 sec + 3,594,194,675 cycles:u # 1.572 GHz + 5,990,655,876 instructions:u # 1.67 insn per cycle + 2.289318334 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1966) (512y: 308) (512z:38918) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_d_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_d_inl1/runTest.exe [ PASSED ] 3 tests. ========================================================================= diff --git a/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_f_inl0.txt b/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_f_inl0.txt index 2417a08fa6..0251bbd4ff 100644 --- a/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_f_inl0.txt +++ b/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_f_inl0.txt @@ -40,45 +40,45 @@ RNDGEN=curdev Building in BUILDDIR=build.512z_f_inl0 for tag=512z_f_inl0_curdev (USEBUILDDIR is set = 1) make: Nothing to be done for `all.512z_f_inl0_curdev'. -DATE: 2021-11-16_14:47:56 +DATE: 2021-12-01_11:35:03 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl0/gcheck.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 7.133814e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 7.139556e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 7.199621e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 7.204622e+05 ) sec^-1 MeanMatrixElemValue = ( 4.059596e+00 +- 2.368053e+00 ) GeV^-4 -TOTAL : 0.551594 sec - 168,335,159 cycles:u # 0.692 GHz - 226,471,160 instructions:u # 1.35 insn per cycle - 0.598689892 seconds time elapsed +TOTAL : 4.858525 sec + 10,369,103,655 cycles:u # 2.154 GHz + 20,824,177,445 instructions:u # 2.01 insn per cycle + 5.138932722 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 255 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl0/gcheck.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 9.249382e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 9.252430e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 9.282397e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 9.285858e+05 ) sec^-1 MeanMatrixElemValue = ( 6.664703e+00 +- 5.072736e+00 ) GeV^-4 -TOTAL : 1.359305 sec - 1,212,991,196 cycles:u # 0.862 GHz - 2,535,586,198 instructions:u # 2.09 insn per cycle - 1.418086640 seconds time elapsed +TOTAL : 1.736571 sec + 1,237,189,827 cycles:u # 0.632 GHz + 2,496,769,062 instructions:u # 2.02 insn per cycle + 2.023754293 seconds time elapsed ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl0/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.801213e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.801213e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.803847e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.803847e+03 ) sec^-1 MeanMatrixElemValue = ( 4.059968e+00 +- 2.367799e+00 ) GeV^-4 -TOTAL : 9.124616 sec - 24,319,272,523 cycles:u # 2.665 GHz - 73,388,316,186 instructions:u # 3.02 insn per cycle - 9.131516670 seconds time elapsed +TOTAL : 9.108182 sec + 24,321,116,779 cycles:u # 2.669 GHz + 73,388,316,467 instructions:u # 3.02 insn per cycle + 9.115275885 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 1303) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl0/runTest.exe @@ -88,14 +88,14 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProce Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 6.580882e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 6.580882e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 6.581704e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 6.581704e+03 ) sec^-1 MeanMatrixElemValue = ( 4.060121e+00 +- 2.367902e+00 ) GeV^-4 -TOTAL : 2.508481 sec - 6,678,866,306 cycles:u # 2.660 GHz - 20,497,996,320 instructions:u # 3.07 insn per cycle - 2.514971966 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 9985) (avx2: 0) (512y: 0) (512z: 0) +TOTAL : 2.508219 sec + 6,691,545,763 cycles:u # 2.664 GHz + 20,504,413,107 instructions:u # 3.06 insn per cycle + 2.514656367 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 9974) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_f_inl0/runTest.exe [ PASSED ] 3 tests. @@ -104,14 +104,14 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProce Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.309736e+04 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.309736e+04 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.314398e+04 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.314398e+04 ) sec^-1 MeanMatrixElemValue = ( 4.059418e+00 +- 2.367906e+00 ) GeV^-4 -TOTAL : 1.269435 sec - 2,869,813,807 cycles:u # 2.255 GHz - 6,998,835,850 instructions:u # 2.44 insn per cycle - 1.275837253 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 8680) (512y: 0) (512z: 0) +TOTAL : 1.264337 sec + 2,868,425,059 cycles:u # 2.262 GHz + 6,999,592,055 instructions:u # 2.44 insn per cycle + 1.270669627 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 8722) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_f_inl0/runTest.exe [ PASSED ] 3 tests. @@ -120,14 +120,14 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProce Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.413195e+04 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.413195e+04 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.414232e+04 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.414232e+04 ) sec^-1 MeanMatrixElemValue = ( 4.059418e+00 +- 2.367906e+00 ) GeV^-4 -TOTAL : 1.177159 sec - 2,668,781,311 cycles:u # 2.261 GHz - 6,471,297,514 instructions:u # 2.42 insn per cycle - 1.183703644 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 8259) (512y: 28) (512z: 0) +TOTAL : 1.176112 sec + 2,669,787,249 cycles:u # 2.263 GHz + 6,472,447,344 instructions:u # 2.42 insn per cycle + 1.182393727 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 8308) (512y: 26) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_f_inl0/runTest.exe [ PASSED ] 3 tests. @@ -136,14 +136,14 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProce Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[16] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.319967e+04 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.319967e+04 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.316884e+04 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.316884e+04 ) sec^-1 MeanMatrixElemValue = ( 4.059419e+00 +- 2.367907e+00 ) GeV^-4 -TOTAL : 1.259206 sec - 1,987,318,670 cycles:u # 1.575 GHz - 3,310,256,608 instructions:u # 1.67 insn per cycle - 1.265681265 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2823) (512y: 13) (512z: 6139) +TOTAL : 1.262979 sec + 1,992,609,610 cycles:u # 1.574 GHz + 3,310,550,374 instructions:u # 1.66 insn per cycle + 1.270179286 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2919) (512y: 12) (512z: 6122) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_f_inl0/runTest.exe [ PASSED ] 3 tests. diff --git a/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_f_inl1.txt b/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_f_inl1.txt index b1050678ef..4abb804da2 100644 --- a/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_f_inl1.txt +++ b/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_f_inl1.txt @@ -1,5 +1,5 @@ -/afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg +/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg OMPFLAGS= AVX=none FPTYPE=f @@ -40,112 +40,112 @@ RNDGEN=curdev Building in BUILDDIR=build.512z_f_inl1 for tag=512z_f_inl1_curdev (USEBUILDDIR is set = 1) make: Nothing to be done for `all.512z_f_inl1_curdev'. -DATE: 2021-10-30_18:35:00 +DATE: 2021-12-01_11:35:38 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl1/gcheck.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl1/gcheck.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 6.755177e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 6.760594e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 6.698727e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 6.703795e+05 ) sec^-1 MeanMatrixElemValue = ( 4.059596e+00 +- 2.368053e+00 ) GeV^-4 -TOTAL : 36.365639 sec - 94,130,351,005 cycles:u # 2.576 GHz - 96,390,605,169 instructions:u # 1.02 insn per cycle - 36.669519084 seconds time elapsed +TOTAL : 36.741951 sec + 94,895,472,348 cycles:u # 2.579 GHz + 96,390,235,215 instructions:u # 1.02 insn per cycle + 37.042689612 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 255 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl1/gcheck.exe -p 2048 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl1/gcheck.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 8.538574e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 8.541988e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 8.539080e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 8.541783e+05 ) sec^-1 MeanMatrixElemValue = ( 6.664703e+00 +- 5.072736e+00 ) GeV^-4 -TOTAL : 1.816556 sec - 1,293,676,002 cycles:u # 0.630 GHz - 2,662,594,694 instructions:u # 2.06 insn per cycle - 2.113360747 seconds time elapsed +TOTAL : 1.809388 sec + 1,319,182,354 cycles:u # 0.647 GHz + 2,675,253,742 instructions:u # 2.03 insn per cycle + 2.102071784 seconds time elapsed ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl1/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl1/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.763046e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.763046e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.765018e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.765018e+03 ) sec^-1 MeanMatrixElemValue = ( 4.059968e+00 +- 2.367799e+00 ) GeV^-4 -TOTAL : 9.322550 sec - 24,873,516,803 cycles:u # 2.667 GHz - 64,759,544,405 instructions:u # 2.60 insn per cycle - 9.330907974 seconds time elapsed +TOTAL : 9.309502 sec + 24,858,794,988 cycles:u # 2.669 GHz + 64,759,551,750 instructions:u # 2.61 insn per cycle + 9.316350169 seconds time elapsed =Symbols in CPPProcess.o= (~sse4:13244) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_f_inl1/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_f_inl1/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 6.074995e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 6.074995e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 6.017002e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 6.017002e+03 ) sec^-1 MeanMatrixElemValue = ( 4.059968e+00 +- 2.367799e+00 ) GeV^-4 -TOTAL : 2.716565 sec - 7,237,758,016 cycles:u # 2.661 GHz - 20,113,715,638 instructions:u # 2.78 insn per cycle - 2.723174385 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4:69018) (avx2: 0) (512y: 0) (512z: 0) +TOTAL : 2.742261 sec + 7,314,780,966 cycles:u # 2.664 GHz + 20,197,668,479 instructions:u # 2.76 insn per cycle + 2.748555485 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4:69323) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_f_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_f_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_f_inl1/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_f_inl1/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.190222e+04 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.190222e+04 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.190092e+04 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.190092e+04 ) sec^-1 MeanMatrixElemValue = ( 4.058922e+00 +- 2.367605e+00 ) GeV^-4 -TOTAL : 1.395121 sec - 3,158,910,056 cycles:u # 2.258 GHz - 6,769,306,523 instructions:u # 2.14 insn per cycle - 1.401670810 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2:47060) (512y: 0) (512z: 0) +TOTAL : 1.395964 sec + 3,161,843,033 cycles:u # 2.261 GHz + 6,774,004,748 instructions:u # 2.14 insn per cycle + 1.402268598 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2:47130) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_f_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_f_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_f_inl1/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_f_inl1/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.098709e+04 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.098709e+04 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.109812e+04 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.109812e+04 ) sec^-1 MeanMatrixElemValue = ( 4.058922e+00 +- 2.367605e+00 ) GeV^-4 -TOTAL : 1.509692 sec - 3,422,149,013 cycles:u # 2.261 GHz - 6,279,290,687 instructions:u # 1.83 insn per cycle - 1.516283502 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2:43307) (512y: 32) (512z: 0) +TOTAL : 1.494450 sec + 3,388,366,334 cycles:u # 2.262 GHz + 6,266,409,846 instructions:u # 1.85 insn per cycle + 1.500646579 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2:43251) (512y: 31) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_f_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_f_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_f_inl1/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_f_inl1/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[16] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.401341e+04 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.401341e+04 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.407201e+04 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.407201e+04 ) sec^-1 MeanMatrixElemValue = ( 4.058923e+00 +- 2.367605e+00 ) GeV^-4 -TOTAL : 1.187543 sec - 1,871,757,256 cycles:u # 1.571 GHz - 3,177,704,632 instructions:u # 1.70 insn per cycle - 1.194067694 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 4094) (512y: 7) (512z:39573) +TOTAL : 1.182769 sec + 1,867,605,024 cycles:u # 1.575 GHz + 3,179,037,328 instructions:u # 1.70 insn per cycle + 1.189243237 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 4162) (512y: 7) (512z:39589) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_f_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_f_inl1/runTest.exe [ PASSED ] 3 tests. ========================================================================= From f16ca21921d28978bc0bf1785269a48877ef4a8f Mon Sep 17 00:00:00 2001 From: Andrea Valassi Date: Wed, 1 Dec 2021 11:48:51 +0100 Subject: [PATCH 22/34] [klas5X] copy ggttgg auto code to manual --- epochX/cudacpp/gg_ttgg/SubProcesses/Makefile | 12 +- epochX/cudacpp/gg_ttgg/SubProcesses/Memory.h | 19 +- .../P1_Sigma_sm_gg_ttxgg/CPPProcess.cc | 82 +++++--- .../P1_Sigma_sm_gg_ttxgg/CPPProcess.h | 14 +- .../P1_Sigma_sm_gg_ttxgg/check_sa.cc | 35 +--- .../cudacpp/gg_ttgg/SubProcesses/runTest.cc | 36 ++-- .../cudacpp/gg_ttgg/SubProcesses/testxxx.cc | 8 +- epochX/cudacpp/gg_ttgg/src/HelAmps_sm.cc | 183 ++++++++++++++---- epochX/cudacpp/gg_ttgg/src/HelAmps_sm.h | 20 +- epochX/cudacpp/gg_ttgg/src/Makefile | 6 +- epochX/cudacpp/gg_ttgg/src/mgOnGpuConfig.h | 84 +++++--- epochX/cudacpp/gg_ttgg/src/mgOnGpuVectors.h | 16 +- epochX/cudacpp/gg_ttgg/src/rambo.cc | 6 +- epochX/cudacpp/gg_ttgg/src/rambo.h | 6 +- 14 files changed, 344 insertions(+), 183 deletions(-) diff --git a/epochX/cudacpp/gg_ttgg/SubProcesses/Makefile b/epochX/cudacpp/gg_ttgg/SubProcesses/Makefile index 69a9a4dbe4..e57c3f2024 100644 --- a/epochX/cudacpp/gg_ttgg/SubProcesses/Makefile +++ b/epochX/cudacpp/gg_ttgg/SubProcesses/Makefile @@ -250,7 +250,7 @@ all.$(TAG): ../../src/$(BUILDDIR)/.build.$(TAG) $(BUILDDIR)/.build.$(TAG) $(cu_m override oldtags=`find $(BUILDDIR) -maxdepth 1 -name '.build.*' ! -name '.build.$(TAG)'` $(BUILDDIR)/.build.$(TAG): - @if [ ! -d $(BUILDDIR) ]; then echo "mkdir $(BUILDDIR)"; mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then echo "mkdir -p $(BUILDDIR)"; mkdir -p $(BUILDDIR); fi @if [ "$(oldtags)" != "" ]; then echo -e "Cannot build for tag=$(TAG) as old builds exist for other tags:\n$(oldtags)\nPlease run 'make clean' first\nIf 'make clean' is not enough: run 'make clean USEBUILDDIR=1 AVX=$(AVX) FPTYPE=$(FPTYPE)' or 'make cleanall'"; exit 1; fi @touch $(BUILDDIR)/.build.$(TAG) @@ -266,28 +266,28 @@ $(LIBDIR)/lib$(MODELLIB).a: ../../src/*.h ../../src/*.cc $(MAKE) -C ../../src $(MAKEDEBUG) $(BUILDDIR)/gcheck_sa.o: gcheck_sa.cu *.h ../../src/*.h # ../../src/*.cu - @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi $(NVCC) $(CPPFLAGS) $(CUFLAGS) -c $< -o $@ $(BUILDDIR)/CPPProcess.o : ../../src/HelAmps_sm.cc $(BUILDDIR)/gCPPProcess.o : ../../src/HelAmps_sm.cc $(BUILDDIR)/%.o : %.cu *.h ../../src/*.h - @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi $(NVCC) $(CPPFLAGS) $(CUFLAGS) -c $< -o $@ # Apply special build flags only to CPPProcess.cc (-flto) #$(BUILDDIR)/CPPProcess.o : CPPProcess.cc *.h ../../src/*.h -# @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi +# @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi # $(CXX) $(CPPFLAGS) $(CXXFLAGS) -flto $(CUINC) -c $< -o $@ ### Apply special build flags only to CPPProcess.cc (AVXFLAGS) ###$(BUILDDIR)/CPPProcess.o : CPPProcess.cc *.h ../../src/*.h -### @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi +### @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi ### $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(AVXFLAGS) $(CUINC) -c $< -o $@ $(BUILDDIR)/%.o : %.cc *.h ../../src/*.h - @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(CUINC) -c $< -o $@ $(cu_main): $(BUILDDIR)/gcheck_sa.o $(LIBDIR)/lib$(MODELLIB).a $(cu_objects) diff --git a/epochX/cudacpp/gg_ttgg/SubProcesses/Memory.h b/epochX/cudacpp/gg_ttgg/SubProcesses/Memory.h index d7d1c7d4da..a9c08d3dd0 100644 --- a/epochX/cudacpp/gg_ttgg/SubProcesses/Memory.h +++ b/epochX/cudacpp/gg_ttgg/SubProcesses/Memory.h @@ -1,8 +1,9 @@ /* - * memory.h + * Memory.h * * Created on: 19.11.2020 * Author: shageboeck + * Modified: avalassi (vectors, alignment) */ #ifndef MEMORY_H @@ -46,13 +47,25 @@ std::unique_ptr> hstMakeUnique(std::size_t N) { #else +template +struct CppHstDeleter { + void operator()(T* mem) { + ::operator delete( mem, std::align_val_t{ mgOnGpu::cppAlign } ); + } +}; + template inline -std::unique_ptr hstMakeUnique(std::size_t N) { return std::unique_ptr{ new T[N]() }; }; +std::unique_ptr> hstMakeUnique(std::size_t N) { + // See https://www.bfilipek.com/2019/08/newnew-align.html#requesting-different-alignment + return std::unique_ptr>{ new( std::align_val_t{ mgOnGpu::cppAlign } ) T[N]() }; +}; #ifdef MGONGPU_CPPSIMD template<> inline -std::unique_ptr hstMakeUnique(std::size_t N) { return std::unique_ptr{ new fptype_v[N/neppV]() }; }; +std::unique_ptr> hstMakeUnique(std::size_t N) { + return std::unique_ptr>{ new( std::align_val_t{ mgOnGpu::cppAlign } ) fptype_v[N/neppV]() }; +}; #endif diff --git a/epochX/cudacpp/gg_ttgg/SubProcesses/P1_Sigma_sm_gg_ttxgg/CPPProcess.cc b/epochX/cudacpp/gg_ttgg/SubProcesses/P1_Sigma_sm_gg_ttxgg/CPPProcess.cc index 164b8af4d7..c4bd7eeb18 100644 --- a/epochX/cudacpp/gg_ttgg/SubProcesses/P1_Sigma_sm_gg_ttxgg/CPPProcess.cc +++ b/epochX/cudacpp/gg_ttgg/SubProcesses/P1_Sigma_sm_gg_ttxgg/CPPProcess.cc @@ -71,10 +71,10 @@ namespace Proc __device__ INLINE void calculate_wavefunctions( int ihel, - const fptype_sv* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM - fptype_sv* allMEs // output: allMEs[npagM][neppM], final |M|^2 averaged over helicities + const fptype* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM + fptype* allMEs // output: allMEs[nevt], |M|^2 running_sum_over_helicities #ifndef __CUDACC__ - , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) #endif ) //ALWAYS_INLINE // attributes are not permitted in a function definition @@ -99,6 +99,9 @@ namespace Proc // === Calculate wavefunctions and amplitudes for all diagrams in all processes - Loop over nevt events === #ifndef __CUDACC__ const int npagV = nevt / neppV; +#ifdef MGONGPU_CPPSIMD + const bool isAligned_allMEs = ( (size_t)(allMEs) % mgOnGpu::cppAlign == 0 ); // require SIMD-friendly alignment by at least neppV*sizeof(fptype) +#endif // ** START LOOP ON IPAGV ** #ifdef _OPENMP // (NB gcc9 or higher, or clang, is required) @@ -106,7 +109,11 @@ namespace Proc // - shared: as the name says // - private: give each thread its own copy, without initialising // - firstprivate: give each thread its own copy, and initialise with value from outside +#ifdef MGONGPU_CPPSIMD +#pragma omp parallel for default(none) shared(allmomenta,allMEs,cHel,cIPC,cIPD,ihel,npagV,isAligned_allMEs) private (amp_sv,w_sv,jamp_sv) +#else #pragma omp parallel for default(none) shared(allmomenta,allMEs,cHel,cIPC,cIPD,ihel,npagV) private (amp_sv,w_sv,jamp_sv) +#endif #endif for ( int ipagV = 0; ipagV < npagV; ++ipagV ) #endif @@ -1913,13 +1920,27 @@ namespace Proc #ifdef __CUDACC__ const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid allMEs[ievt] += deltaMEs; - //printf( "calculate_wavefunction: %6d %2d %f\n", ievt, ihel, allMEs[ievt] ); + //if ( cNGoodHel > 0 ) printf( "calculate_wavefunction: %6d %2d %f\n", ievt, ihel, allMEs[ievt] ); +#else +#ifdef MGONGPU_CPPSIMD + if ( isAligned_allMEs ) + { + *reinterpret_cast( &( allMEs[ipagV*neppV] ) ) += deltaMEs; + } + else + { + for ( int ieppV=0; ieppV 0 ) + // for ( int ieppV=0; ieppV 0 ) printf( "calculate_wavefunction: %6d %2d %f\n", ipagV, ihel, allMEs[ipagV] ); +#endif #endif } - mgDebug( 1, __FUNCTION__ ); return; } @@ -2011,10 +2032,13 @@ namespace Proc memcpy( cHel, tHel, ncomb * mgOnGpu::npar * sizeof(short) ); #endif // SANITY CHECK: GPU memory usage may be based on casts of fptype[2] to cxtype - assert( sizeof(cxtype) == 2 * sizeof(fptype) ); + static_assert( sizeof(cxtype) == 2 * sizeof(fptype) ); #ifndef __CUDACC__ - // SANITY CHECK: momenta AOSOA uses vectors with the same size as fptype_v - assert( neppV == mgOnGpu::neppM ); + // SANITY CHECK: check that neppR, neppM and neppV are powers of two (https://stackoverflow.com/a/108360) + auto ispoweroftwo = []( int n ) { return ( n > 0 ) && !( n & ( n - 1 ) ); }; + static_assert( ispoweroftwo( mgOnGpu::neppR ) ); + static_assert( ispoweroftwo( mgOnGpu::neppM ) ); + static_assert( ispoweroftwo( neppV ) ); #endif } @@ -2126,9 +2150,9 @@ namespace Proc #ifdef __CUDACC__ __global__ - void sigmaKin_getGoodHel( const fptype_sv* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM - fptype_sv* allMEs, // output: allMEs[npagM][neppM], final |M|^2 averaged over helicities - bool* isGoodHel ) // output: isGoodHel[ncomb] - device array + void sigmaKin_getGoodHel( const fptype* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM + fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities + bool* isGoodHel ) // output: isGoodHel[ncomb] - device array { const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid // FIXME: assume process.nprocesses == 1 for the moment (eventually: need a loop over processes here?) @@ -2146,33 +2170,35 @@ namespace Proc } } #else - void sigmaKin_getGoodHel( const fptype_sv* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM - fptype_sv* allMEs, // output: allMEs[npagM][neppM], final |M|^2 averaged over helicities - bool* isGoodHel // output: isGoodHel[ncomb] - device array - , const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + void sigmaKin_getGoodHel( const fptype* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM + fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities + bool* isGoodHel // output: isGoodHel[ncomb] - device array + , const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) { + assert( (size_t)(allmomenta) % mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment + //assert( (size_t)(allMEs) % mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment const int maxtry0 = ( neppV > 16 ? neppV : 16 ); // 16, but at least neppV (otherwise the npagV loop does not even start) - fptype_sv allMEsLast[maxtry0/neppV] = { 0 }; + fptype allMEsLast[maxtry0] = { 0 }; const int maxtry = std::min( maxtry0, nevt ); // 16, but at most nevt (avoid invalid memory access if nevt allMEs[nevt*nprocesses]?) __global__ - void sigmaKin( const fptype_sv* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM - fptype_sv* allMEs // output: allMEs[npagM][neppM], final |M|^2 averaged over helicities + void sigmaKin( const fptype* allmomenta, // input: momenta as AOSOA[npagM][npar][4][neppM] with nevt=npagM*neppM + fptype* allMEs // output: allMEs[nevt], |M|^2 final_avg_over_helicities #ifndef __CUDACC__ - , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) #endif ) { @@ -2242,7 +2268,8 @@ namespace Proc const int npagV = nevt/neppV; for ( int ipagV = 0; ipagV < npagV; ++ipagV ) { - allMEs[ipagV] = fptype_sv{ 0 }; + for ( int ieppV=0; ieppV( nRnarray ); // AOSOA[npagR][nparf][np4][neppR] (NB: nevt=npagR*neppR) + auto hstRnarray = hstMakeUnique( nRnarray ); // AOSOA[npagR][nparf][np4][neppR] (NB: nevt=npagR*neppR) #endif - auto hstMomenta = hstMakeUnique( nMomenta ); // AOSOA[npagM][npar][np4][neppM] (NB: nevt=npagM*neppM) - auto hstIsGoodHel = hstMakeUnique( ncomb ); - auto hstWeights = hstMakeUnique( nWeights ); - auto hstMEs = hstMakeUnique( nMEs ); // AOSOA[npagM][neppM] (NB: nevt=npagM*neppM) + auto hstMomenta = hstMakeUnique( nMomenta ); // AOSOA[npagM][npar][np4][neppM] (NB: nevt=npagM*neppM) + auto hstIsGoodHel = hstMakeUnique( ncomb ); + auto hstWeights = hstMakeUnique( nWeights ); + auto hstMEs = hstMakeUnique( nMEs ); // ARRAY[nevt] #ifdef __CUDACC__ - auto devRnarray = devMakeUnique( nRnarray ); // AOSOA[npagR][nparf][np4][neppR] (NB: nevt=npagR*neppR) - auto devMomenta = devMakeUnique( nMomenta ); // AOSOA[npagM][npar][np4][neppM] (NB: nevt=npagM*neppM) - auto devIsGoodHel = devMakeUnique( ncomb ); - auto devWeights = devMakeUnique( nWeights ); - auto devMEs = devMakeUnique( nMEs ); // AOSOA[npagM][neppM] (NB: nevt=npagM*neppM) + auto devRnarray = devMakeUnique( nRnarray ); // AOSOA[npagR][nparf][np4][neppR] (NB: nevt=npagR*neppR) + auto devMomenta = devMakeUnique( nMomenta ); // AOSOA[npagM][npar][np4][neppM] (NB: nevt=npagM*neppM) + auto devIsGoodHel = devMakeUnique( ncomb ); + auto devWeights = devMakeUnique( nWeights ); + auto devMEs = devMakeUnique( nMEs ); // ARRAY[nevt] #if defined MGONGPU_CURAND_ONHOST or defined MGONGPU_COMMONRAND_ONHOST const int nbytesRnarray = nRnarray * sizeof(fptype); @@ -553,37 +553,22 @@ int main(int argc, char **argv) // NB: 'setw' affects only the next field (of any type) std::cout << std::scientific // fixed format: affects all floats (default precision: 6) << std::setw(4) << ipar + 1 -#ifndef MGONGPU_CPPSIMD << std::setw(14) << hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + 0*neppM + ieppM] // AOSOA[ipagM][ipar][0][ieppM] << std::setw(14) << hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + 1*neppM + ieppM] // AOSOA[ipagM][ipar][1][ieppM] << std::setw(14) << hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + 2*neppM + ieppM] // AOSOA[ipagM][ipar][2][ieppM] << std::setw(14) << hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + 3*neppM + ieppM] // AOSOA[ipagM][ipar][3][ieppM] -#else - << std::setw(14) << hstMomenta[ipagM*npar*np4 + ipar*np4 + 0][ieppM] // AOSOA[ipagM][ipar][0][ieppM] - << std::setw(14) << hstMomenta[ipagM*npar*np4 + ipar*np4 + 1][ieppM] // AOSOA[ipagM][ipar][1][ieppM] - << std::setw(14) << hstMomenta[ipagM*npar*np4 + ipar*np4 + 2][ieppM] // AOSOA[ipagM][ipar][2][ieppM] - << std::setw(14) << hstMomenta[ipagM*npar*np4 + ipar*np4 + 3][ieppM] // AOSOA[ipagM][ipar][3][ieppM] -#endif << std::endl << std::defaultfloat; // default format: affects all floats } std::cout << std::string(SEP79, '-') << std::endl; // Display matrix elements std::cout << " Matrix element = " -#ifndef MGONGPU_CPPSIMD << hstMEs[ievt] -#else - << hstMEs[ievt/neppM][ievt%neppM] -#endif << " GeV^" << meGeVexponent << std::endl; // FIXME: assume process.nprocesses == 1 std::cout << std::string(SEP79, '-') << std::endl; } // Fill the arrays with ALL MEs and weights -#ifndef MGONGPU_CPPSIMD matrixelementALL[iiter*nevt + ievt] = hstMEs[ievt]; // FIXME: assume process.nprocesses == 1 -#else - matrixelementALL[iiter*nevt + ievt] = hstMEs[ievt/neppM][ievt%neppM]; // FIXME: assume process.nprocesses == 1 -#endif weightALL[iiter*nevt + ievt] = hstWeights[ievt]; } diff --git a/epochX/cudacpp/gg_ttgg/SubProcesses/runTest.cc b/epochX/cudacpp/gg_ttgg/SubProcesses/runTest.cc index d92ffe4e7a..a4f3519434 100644 --- a/epochX/cudacpp/gg_ttgg/SubProcesses/runTest.cc +++ b/epochX/cudacpp/gg_ttgg/SubProcesses/runTest.cc @@ -21,7 +21,7 @@ template using unique_ptr_host = std::unique_ptr>; #else template -using unique_ptr_host = std::unique_ptr; +using unique_ptr_host = std::unique_ptr>; #endif struct CUDA_CPU_TestBase : public TestDriverBase { @@ -47,11 +47,11 @@ struct CPUTest : public CUDA_CPU_TestBase { // Struct data members (process, and memory structures for random numbers, momenta, matrix elements and weights on host and device) // [NB the hst/dev memory arrays must be initialised in the constructor, see issue #290] Proc::CPPProcess process; - unique_ptr_host hstRnarray; - unique_ptr_host hstMomenta; - unique_ptr_host hstIsGoodHel; - unique_ptr_host hstWeights; - unique_ptr_host hstMEs; + unique_ptr_host hstRnarray; + unique_ptr_host hstMomenta; + unique_ptr_host hstIsGoodHel; + unique_ptr_host hstWeights; + unique_ptr_host hstMEs; // Create a process object // Read param_card and set parameters @@ -61,11 +61,11 @@ struct CPUTest : public CUDA_CPU_TestBase { CPUTest( const std::string& refFileName ) : CUDA_CPU_TestBase( refFileName ), process(niter, gpublocks, gputhreads, /*verbose=*/false), - hstRnarray { hstMakeUnique( nRnarray ) }, // AOSOA[npagR][nparf][np4][neppR] - hstMomenta { hstMakeUnique( nMomenta ) }, // AOSOA[npagM][npar][np4][neppM] - hstIsGoodHel{ hstMakeUnique( mgOnGpu::ncomb ) }, - hstWeights { hstMakeUnique( nWeights ) }, - hstMEs { hstMakeUnique( nMEs ) } // AOSOA[npagM][neppM] + hstRnarray { hstMakeUnique( nRnarray ) }, // AOSOA[npagR][nparf][np4][neppR] + hstMomenta { hstMakeUnique( nMomenta ) }, // AOSOA[npagM][npar][np4][neppM] + hstIsGoodHel{ hstMakeUnique( mgOnGpu::ncomb ) }, + hstWeights { hstMakeUnique( nWeights ) }, + hstMEs { hstMakeUnique( nMEs ) } // ARRAY[nevt] { process.initProc("../../Cards/param_card.dat"); } @@ -106,19 +106,11 @@ struct CPUTest : public CUDA_CPU_TestBase { assert(particle < npar); const auto ipagM = evtNo / neppM; // #eventpage in this iteration const auto ieppM = evtNo % neppM; // #event in the current eventpage in this iteration -#ifndef MGONGPU_CPPSIMD return hstMomenta[ipagM*npar*np4*neppM + particle*np4*neppM + component*neppM + ieppM]; -#else - return hstMomenta[ipagM*npar*np4 + particle*np4 + component][ieppM]; -#endif }; fptype getMatrixElement(std::size_t ievt) const override { -#ifndef MGONGPU_CPPSIMD return hstMEs[ievt]; -#else - return hstMEs[ievt/neppV][ievt%neppV]; -#endif } }; @@ -161,12 +153,12 @@ struct CUDATest : public CUDA_CPU_TestBase { hstMomenta { hstMakeUnique( nMomenta ) }, // AOSOA[npagM][npar][np4][neppM] (nevt=npagM*neppM) hstIsGoodHel{ hstMakeUnique( mgOnGpu::ncomb ) }, hstWeights { hstMakeUnique( nWeights ) }, - hstMEs { hstMakeUnique( nMEs ) }, + hstMEs { hstMakeUnique( nMEs ) }, // ARRAY[nevt] devRnarray { devMakeUnique( nRnarray ) }, // AOSOA[npagR][nparf][np4][neppR] (nevt=npagR*neppR) - devMomenta { devMakeUnique( nMomenta ) }, + devMomenta { devMakeUnique( nMomenta ) }, // AOSOA[npagM][npar][np4][neppM] (nevt=npagM*neppM) devIsGoodHel{ devMakeUnique( mgOnGpu::ncomb ) }, devWeights { devMakeUnique( nWeights ) }, - devMEs { devMakeUnique( nMEs ) } + devMEs { devMakeUnique( nMEs ) } // ARRAY[nevt] { process.initProc("../../Cards/param_card.dat"); } diff --git a/epochX/cudacpp/gg_ttgg/SubProcesses/testxxx.cc b/epochX/cudacpp/gg_ttgg/SubProcesses/testxxx.cc index 6df896c75f..9ae06a7404 100644 --- a/epochX/cudacpp/gg_ttgg/SubProcesses/testxxx.cc +++ b/epochX/cudacpp/gg_ttgg/SubProcesses/testxxx.cc @@ -31,8 +31,8 @@ TEST( XTESTID_CPU( MG_EPOCH_PROCESS_ID ), testxxx ) assert( nevt % neppM == 0 ); // nevt must be a multiple of neppM // Fill in the input momenta const int nMomenta = np4 * npar * nevt; - auto hstMomenta = hstMakeUnique( nMomenta ); // AOSOA[npagM][npar=4][np4=4][neppM] - const fptype par0[np4 * nevt] = // AOS[nevt][np4] + auto hstMomenta = hstMakeUnique( nMomenta ); // AOSOA[npagM][npar=4][np4=4][neppM] + const fptype par0[np4 * nevt] = // AOS[nevt][np4] { 500, 0, 0, 500, // #0 (m=0 pT=0 E=pz>0) 500, 0, 0, -500, // #1 (m=0 pT=0 -E=pz<0) 500, 300, 400, 0, // #2 (m=0 pT>0 pz=0) @@ -72,11 +72,7 @@ TEST( XTESTID_CPU( MG_EPOCH_PROCESS_ID ), testxxx ) { const int ipagM = ievt/neppM; // #eventpage in this iteration const int ieppM = ievt%neppM; // #event in the current eventpage in this iteration -#ifdef MGONGPU_CPPSIMD - hstMomenta[ipagM*npar*np4 + ipar*np4 + ip4][ieppM] = par0[ievt*np4 + ip4]; // AOS to AOSOA -#else hstMomenta[ipagM*npar*np4*neppM + ipar*np4*neppM + ip4*neppM + ieppM] = par0[ievt*np4 + ip4]; // AOS to AOSOA -#endif } } // Expected output wavefunctions diff --git a/epochX/cudacpp/gg_ttgg/src/HelAmps_sm.cc b/epochX/cudacpp/gg_ttgg/src/HelAmps_sm.cc index 62dbaa3768..b70f901581 100644 --- a/epochX/cudacpp/gg_ttgg/src/HelAmps_sm.cc +++ b/epochX/cudacpp/gg_ttgg/src/HelAmps_sm.cc @@ -23,8 +23,8 @@ namespace MG5_sm //-------------------------------------------------------------------------- -#ifdef __CUDACC__ - // Return by reference + // Decode momentum AOSOA: compute address of fptype for the given particle, 4-momentum component and event + // Return the fptype by reference (equivalent to returning its memory address) __device__ inline const fptype& pIparIp4Ievt( const fptype* momenta, // input: momenta as AOSOA[npagM][npar][4][neppM] const int ipar, @@ -33,38 +33,149 @@ namespace MG5_sm { using mgOnGpu::np4; using mgOnGpu::npar; - const int neppM = mgOnGpu::neppM; // AOSOA layout: constant at compile-time - const int ipagM = ievt/neppM; // #eventpage in this iteration - const int ieppM = ievt%neppM; // #event in the current eventpage in this iteration - //printf( "%f\n", momenta[ipagM*npar*np4*neppM + ipar*np4*neppM + ip4*neppM + ieppM] ); + constexpr int neppM = mgOnGpu::neppM; // AOSOA layout: constant at compile-time + const int ipagM = ievt/neppM; // #event "M-page" + const int ieppM = ievt%neppM; // #event in the current event M-page + //printf( "%2d %2d %8d %8.3f\n", ipar, ip4, ievt, momenta[ipagM*npar*np4*neppM + ipar*np4*neppM + ip4*neppM + ieppM] ); return momenta[ipagM*npar*np4*neppM + ipar*np4*neppM + ip4*neppM + ieppM]; // AOSOA[ipagM][ipar][ip4][ieppM] //fptype (*momenta)[npar][np4][neppM] = (fptype (*)[npar][np4][neppM]) momenta; // cast to multiD array pointer (AOSOA) //return momenta[ipagM][ipar][ip4][ieppM]; // this seems ~1-2% faster in eemumu C++? } -#else - // Return by value: it seems a tiny bit faster than returning a reference (both for scalar and vector), not clear why + +#ifndef __CUDACC__ + // Return a SIMD vector of fptype's for neppV events (for the given particle, 4-momentum component and event "V-page") + // Return the vector by value: strictly speaking, this is only unavoidable for neppM=neppV (both being powers of 2), the momentum neppM-AOSOA is reinterpreted in terms of neppV-vectors: + // it could also be returned by reference, but no performance degradation is observed when returning by value inline - fptype_sv pIparIp4Ipag( const fptype_sv* momenta, // input: momenta as AOSOA[npagM][npar][4][neppM] + fptype_sv pIparIp4Ipag( const fptype* momenta, // input: momenta as AOSOA[npagM][npar][4][neppM] const int ipar, const int ip4, - const int ipagM ) + const int ipagV ) { - /* - //#ifndef MGONGPU_CPPSIMD - // TEMPORARY during epochX step3! Eventually remove this section (start) - // TEMPORARY during epochX step3! THERE IS NO SIMD YET: HENCE ipagM=ievt - // NB: this is needed for neppM>1 while neppV==1 (no SIMD) in eemumu.auto - // NB: this remains valid for neppM==1 with neppV==1 in eemumu (scalar) - return pIparIp4Ievt( momenta, ipar, ip4, ipagM ); - // TEMPORARY during epochX step3! Eventually remove this section (end) - //#else - */ - // NB: this assumes that neppV == neppM! - // NB: this is the same as "pIparIp4Ievt( momenta, ipar, ip4, ipagM )" for neppM==1 with neppV==1 - using mgOnGpu::np4; - using mgOnGpu::npar; - //printf( "%f\n", momenta[ipagM*npar*np4 + ipar*np4 + ip4] ); - return momenta[ipagM*npar*np4 + ipar*np4 + ip4]; // AOSOA[ipagM][ipar][ip4][ieppM] + const int ievt0 = ipagV*neppV; // virtual event V-page ipagV contains neppV events [ievt0...ievt0+neppV-1] +#ifdef MGONGPU_CPPSIMD + constexpr int neppM = mgOnGpu::neppM; // AOSOA layout: constant at compile-time + // Use c++17 "if constexpr": compile-time branching + if constexpr ( ( neppM >= neppV ) && ( neppM%neppV == 0 ) ) + { + constexpr bool useReinterpretCastIfPossible = true; // DEFAULT + //constexpr bool useReinterpretCastIfPossible = false; // FOR PERFORMANCE TESTS + constexpr bool skipAlignmentCheck = true; // DEFAULT (MAY SEGFAULT, NEEDS A SANITY CHECK ELSEWHERE!) + //constexpr bool skipAlignmentCheck = false; // SLOWER BUT SAFER + if constexpr ( useReinterpretCastIfPossible && skipAlignmentCheck ) + { + //static bool first=true; if( first ){ std::cout << "WARNING! skip alignment check" << std::endl; first=false; } // SLOWS DOWN... + // Fastest (4.92E6 in eemumu 512y) + // This requires alignment for momenta - segmentation fault otherwise! + return *reinterpret_cast( &( pIparIp4Ievt( momenta, ipar, ip4, ievt0 ) ) ); + } + else if ( useReinterpretCastIfPossible && ( (size_t)(momenta) % mgOnGpu::cppAlign == 0 ) ) + { + //static bool first=true; if( first ){ std::cout << "WARNING! alignment ok, use reinterpret cast" << std::endl; first=false; } // SLOWS DOWN... + // A bit (6%) slower (4.62E6 in eemumu 512y) because of the alignment check + // This explicitly checks alignment for momenta to avoid segmentation faults + return *reinterpret_cast( &( pIparIp4Ievt( momenta, ipar, ip4, ievt0 ) ) ); + } + else + { + //static bool first=true; if( first ){ std::cout << "WARNING! AOSOA but no reinterpret cast" << std::endl; first=false; } // SLOWS DOWN... + // A bit (2%) slower (4.86E6 in eemumu 512y) + // This does not require alignment for momenta, but it requires AOSOA with neppM>=neppV and neppM%neppV==0 + const fptype* out0 = &( pIparIp4Ievt( momenta, ipar, ip4, ievt0) ); +#if MGONGPU_CPPSIMD == 2 + return fptype_v{ *( out0 ), + *( out0+1 ) }; +#elif MGONGPU_CPPSIMD == 4 + return fptype_v{ *( out0 ), + *( out0+1 ), + *( out0+2 ), + *( out0+3 ) }; +#elif MGONGPU_CPPSIMD == 8 + return fptype_v{ *( out0 ), + *( out0+1 ), + *( out0+2 ), + *( out0+3 ), + *( out0+4 ), + *( out0+5 ), + *( out0+6 ), + *( out0+7 ) }; +#elif MGONGPU_CPPSIMD == 16 + return fptype_v{ *( out0 ), + *( out0+1 ), + *( out0+2 ), + *( out0+3 ), + *( out0+4 ), + *( out0+5 ), + *( out0+6 ), + *( out0+7 ), + *( out0+8 ), + *( out0+9 ), + *( out0+10 ), + *( out0+11 ), + *( out0+12 ), + *( out0+13 ), + *( out0+14 ), + *( out0+15 ) }; +#else +#warning Internal error? This code should never be reached! + // A bit (2%) slower (4.86E6 in eemumu 512y) + // This does not require alignment for momenta, but it requires AOSOA with neppM>=neppV and neppM%neppV==0 + fptype_v out; + for ( int ieppV=0; ieppV=neppV and neppM%neppV==0 (e.g. can be used with AOS neppM==1) +#if MGONGPU_CPPSIMD == 2 + return fptype_v{ pIparIp4Ievt( momenta, ipar, ip4, ievt0 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+1 ) }; +#elif MGONGPU_CPPSIMD == 4 + return fptype_v{ pIparIp4Ievt( momenta, ipar, ip4, ievt0 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+1 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+2 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+3 ) }; +#elif MGONGPU_CPPSIMD == 8 + return fptype_v{ pIparIp4Ievt( momenta, ipar, ip4, ievt0 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+1 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+2 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+3 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+4 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+5 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+6 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+7 ) }; +#elif MGONGPU_CPPSIMD == 16 + return fptype_v{ pIparIp4Ievt( momenta, ipar, ip4, ievt0 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+1 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+2 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+3 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+4 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+5 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+6 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+7 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+8 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+9 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+10 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+11 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+12 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+13 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+14 ), + pIparIp4Ievt( momenta, ipar, ip4, ievt0+15 ) }; +#else +#warning Internal error? This code should never be reached! + // Much much (40%) slower (3.02E6 in eemumu 512y) + fptype_v out; + for ( int ieppV=0; ieppV 0) __device__ - void ipzxxx( const fptype_sv* momenta, + void ipzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -257,7 +368,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) __device__ - void imzxxx( const fptype_sv* momenta, + void imzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -306,7 +417,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PT > 0) __device__ - void ixzxxx( const fptype_sv* momenta, + void ixzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -367,7 +478,7 @@ namespace MG5_sm // Compute the output wavefunction vc[6] from the input momenta[npar*4*nevt] __device__ - void vxxxxx( const fptype_sv* momenta, + void vxxxxx( const fptype* momenta, const fptype vmass, // input: vector boson mass const int nhel, // input: -1, 0 (only if vmass!=0) or +1 (helicity of vector boson) const int nsv, // input: +1 (final) or -1 (initial) @@ -503,7 +614,7 @@ namespace MG5_sm // Compute the output wavefunction sc[3] from the input momenta[npar*4*nevt] __device__ - void sxxxxx( const fptype_sv* momenta, + void sxxxxx( const fptype* momenta, const fptype, // WARNING: input "smass" unused (missing in Fortran) - scalar boson mass const int, // WARNING: input "nhel" unused (missing in Fortran) - scalar has no helicity! const int nss, // input: +1 (final) or -1 (initial) @@ -544,7 +655,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] __device__ - void oxxxxx( const fptype_sv* momenta, + void oxxxxx( const fptype* momenta, const fptype fmass, // input: fermion mass const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -681,7 +792,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) __device__ - void opzxxx( const fptype_sv* momenta, + void opzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -730,7 +841,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) __device__ - void omzxxx( const fptype_sv* momenta, + void omzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -782,7 +893,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PT > 0) __device__ - void oxzxxx( const fptype_sv* momenta, + void oxzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) diff --git a/epochX/cudacpp/gg_ttgg/src/HelAmps_sm.h b/epochX/cudacpp/gg_ttgg/src/HelAmps_sm.h index 087a6d4cdd..529f37c33d 100644 --- a/epochX/cudacpp/gg_ttgg/src/HelAmps_sm.h +++ b/epochX/cudacpp/gg_ttgg/src/HelAmps_sm.h @@ -28,7 +28,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] __device__ INLINE - void ixxxxx( const fptype_sv* momenta, + void ixxxxx( const fptype* momenta, const fptype fmass, // input: fermion mass const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -43,7 +43,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) __device__ INLINE - void ipzxxx( const fptype_sv* momenta, + void ipzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -58,7 +58,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) __device__ INLINE - void imzxxx( const fptype_sv* momenta, + void imzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -73,7 +73,7 @@ namespace MG5_sm // Compute the output wavefunction fi[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PT > 0) __device__ INLINE - void ixzxxx( const fptype_sv* momenta, + void ixzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -87,7 +87,7 @@ namespace MG5_sm // Compute the output wavefunction vc[6] from the input momenta[npar*4*nevt] __device__ INLINE - void vxxxxx( const fptype_sv* momenta, + void vxxxxx( const fptype* momenta, const fptype vmass, // input: vector boson mass const int nhel, // input: -1, 0 (only if vmass!=0) or +1 (helicity of vector boson) const int nsv, // input: +1 (final) or -1 (initial) @@ -101,7 +101,7 @@ namespace MG5_sm // Compute the output wavefunction sc[3] from the input momenta[npar*4*nevt] __device__ INLINE - void sxxxxx( const fptype_sv* momenta, + void sxxxxx( const fptype* momenta, const fptype, // WARNING: input "smass" unused (missing in Fortran) - scalar boson mass const int, // WARNING: input "nhel" unused (missing in Fortran) - scalar has no helicity! const int nss, // input: +1 (final) or -1 (initial) @@ -115,7 +115,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] __device__ INLINE - void oxxxxx( const fptype_sv* momenta, + void oxxxxx( const fptype* momenta, const fptype fmass, // input: fermion mass const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -130,7 +130,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) __device__ INLINE - void opzxxx( const fptype_sv* momenta, + void opzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -145,7 +145,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] // ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) __device__ INLINE - void omzxxx( const fptype_sv* momenta, + void omzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) @@ -159,7 +159,7 @@ namespace MG5_sm // Compute the output wavefunction fo[6] from the input momenta[npar*4*nevt] __device__ INLINE - void oxzxxx( const fptype_sv* momenta, + void oxzxxx( const fptype* momenta, //const fptype fmass, // ASSUME fermion mass==0 const int nhel, // input: -1 or +1 (helicity of fermion) const int nsf, // input: +1 (particle) or -1 (antiparticle) diff --git a/epochX/cudacpp/gg_ttgg/src/Makefile b/epochX/cudacpp/gg_ttgg/src/Makefile index ce74470562..6e6b6a0c39 100644 --- a/epochX/cudacpp/gg_ttgg/src/Makefile +++ b/epochX/cudacpp/gg_ttgg/src/Makefile @@ -160,7 +160,7 @@ all.$(TAG): $(BUILDDIR)/.build.$(TAG) $(target) override oldtags=`find $(BUILDDIR) -maxdepth 1 -name '.build.*' ! -name '.build.$(TAG)'` $(BUILDDIR)/.build.$(TAG): - @if [ ! -d $(BUILDDIR) ]; then echo "mkdir $(BUILDDIR)"; mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then echo "mkdir -p $(BUILDDIR)"; mkdir -p $(BUILDDIR); fi @if [ "$(oldtags)" != "" ]; then echo -e "Cannot build for tag=$(TAG) as old builds exist for other tags:\n$(oldtags)\nPlease run 'make clean' first\nIf 'make clean' is not enough: run 'make clean USEBUILDDIR=1 AVX=$(AVX) FPTYPE=$(FPTYPE)' or 'make cleanall'"; exit 1; fi @touch $(BUILDDIR)/.build.$(TAG) @@ -171,11 +171,11 @@ debug: $(target) # NB: cuda includes are needed in the C++ code for curand.h $(BUILDDIR)/%.o : %.cc *.h - @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi + @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(CUINC) -c $< -o $@ #$(BUILDDIR)/%.o : %.cu *.h -# @if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi +# @if [ ! -d $(BUILDDIR) ]; then mkdir -p $(BUILDDIR); fi # $(NVCC) $(CPPFLAGS) $(CUFLAGS) -c $< -o $@ $(target): $(cxx_objects) $(cu_objects) diff --git a/epochX/cudacpp/gg_ttgg/src/mgOnGpuConfig.h b/epochX/cudacpp/gg_ttgg/src/mgOnGpuConfig.h index ab498c8f10..c2b6f6bacb 100644 --- a/epochX/cudacpp/gg_ttgg/src/mgOnGpuConfig.h +++ b/epochX/cudacpp/gg_ttgg/src/mgOnGpuConfig.h @@ -96,47 +96,79 @@ namespace mgOnGpu //const int ntpbMAX = 256; // AV Apr2021: why had I set this to 256? const int ntpbMAX = 1024; // NB: 512 is ok, but 1024 does fail with "too many resources requested for launch" - // Vector sizes for AOSOA memory layouts (GPU coalesced memory access, CPU SIMD vectorization) - // (these are all best kept as a compile-time constants: see issue #23) + // Alignment requirement for using reinterpret_cast with SIMD vectorized code + // (using reinterpret_cast with non aligned memory may lead to segmentation faults!) +#ifndef __CUDACC__ + constexpr int cppAlign = 64; // alignment requirement for SIMD vectorization (64-byte i.e. 512-bit) +#endif - // Number of Events Per Page in the momenta AOSOA memory layout + // C++ SIMD vectorization width (this will be used to set neppV) #ifdef __CUDACC__ #undef MGONGPU_CPPSIMD - // ----------------------------------------------------------------------------------------- - // --- GPUs: neppM must be a power of 2 times the number of fptype's in a 32-byte cacheline - // --- This is relevant to ensure coalesced access to momenta in global memory - // --- Note that neppR is hardcoded and may differ from neppM and neppV on some platforms - // ----------------------------------------------------------------------------------------- - //const int neppM = 64/sizeof(fptype); // 2x 32-byte GPU cache lines: 8 (DOUBLE) or 16 (FLOAT) - const int neppM = 32/sizeof(fptype); // (DEFAULT) 32-byte GPU cache line: 4 (DOUBLE) or 8 (FLOAT) - //const int neppM = 1; // *** NB: this is equivalent to AOS *** - //const int neppM = 32; // older default #else - // ----------------------------------------------------------------------------------------- - // --- CPUs: neppM must be exactly equal to the number of fptype's in a vector register - // --- [DEFAULT is 256-width AVX512 aka "512y" on CPUs, 32-byte as GPUs, faster than AVX2] - // --- The logic of the code requires the size neppV of fptype_v to be equal to neppM - // --- Note that neppR is hardcoded and may differ from neppM and neppV on some platforms - // ----------------------------------------------------------------------------------------- #if defined __AVX512VL__ -#define MGONGPU_CPPSIMD 1 #ifdef MGONGPU_PVW512 - const int neppM = 64/sizeof(fptype); // "512z" AVX512 with 512 width (512-bit ie 64-byte): 8 (DOUBLE) or 16 (FLOAT) + // "512z" AVX512 with 512 width (512-bit ie 64-byte): 8 (DOUBLE) or 16 (FLOAT) +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 8 #else - const int neppM = 32/sizeof(fptype); // "512y" AVX512 with 256 width (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [gcc DEFAULT] +#define MGONGPU_CPPSIMD 16 +#endif +#else + // "512y" AVX512 with 256 width (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [gcc DEFAULT] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 4 +#else +#define MGONGPU_CPPSIMD 8 +#endif #endif #elif defined __AVX2__ -#define MGONGPU_CPPSIMD 1 - const int neppM = 32/sizeof(fptype); // "avx2" AVX2 (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [clang DEFAULT] + // "avx2" AVX2 (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [clang DEFAULT] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 4 +#else +#define MGONGPU_CPPSIMD 8 +#endif #elif defined __SSE4_2__ -#define MGONGPU_CPPSIMD 1 - const int neppM = 16/sizeof(fptype); // "sse4" SSE4.2 (128-bit ie 16-byte): 2 (DOUBLE) or 4 (FLOAT) + // "sse4" SSE4.2 (128-bit ie 16-byte): 2 (DOUBLE) or 4 (FLOAT) +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 2 #else +#define MGONGPU_CPPSIMD 4 +#endif +#else + // "none" i.e. no SIMD (*** NB: this is equivalent to AOS ***) #undef MGONGPU_CPPSIMD - const int neppM = 1; // "none" i.e. no SIMD (*** NB: this is equivalent to AOS ***) #endif #endif + // Number of Events Per Page in the momenta AOSOA memory layout +#ifdef __CUDACC__ + // ----------------------------------------------------------------------------------------------- + // --- GPUs: neppM is best set to a power of 2 times the number of fptype's in a 32-byte cacheline + // --- This is relevant to ensure coalesced access to momenta in global memory + // --- Note that neppR is hardcoded and may differ from neppM and neppV on some platforms + // ----------------------------------------------------------------------------------------------- + //const int neppM = 64/sizeof(fptype); // 2x 32-byte GPU cache lines (512 bits): 8 (DOUBLE) or 16 (FLOAT) + const int neppM = 32/sizeof(fptype); // (DEFAULT) 32-byte GPU cache line (256 bits): 4 (DOUBLE) or 8 (FLOAT) + //const int neppM = 1; // *** NB: this is equivalent to AOS *** +#else + // ----------------------------------------------------------------------------------------------- + // --- CPUs: neppM is best set equal to the number of fptype's (neppV) in a vector register + // --- This is relevant to ensure faster access to momenta from C++ memory cache lines + // --- However, neppM is now decoupled from neppV (issue #176) and can be separately hardcoded + // --- In practice, neppR, neppM and neppV could now (in principle) all be different + // ----------------------------------------------------------------------------------------------- +#ifdef MGONGPU_CPPSIMD + const int neppM = MGONGPU_CPPSIMD; // (DEFAULT) neppM=neppV for optimal performance +#else + const int neppM = 1; // (DEFAULT) neppM=neppV for optimal performance (NB: this is equivalent to AOS) +#endif + //const int neppM = 64/sizeof(fptype); // maximum CPU vector width (512 bits): 8 (DOUBLE) or 16 (FLOAT) + //const int neppM = 32/sizeof(fptype); // lower CPU vector width (256 bits): 4 (DOUBLE) or 8 (FLOAT) + //const int neppM = 1; // *** NB: this is equivalent to AOS *** +#endif + // Number of Events Per Page in the random number AOSOA memory layout // *** NB Different values of neppR lead to different physics results: the *** // *** same 1d array is generated, but it is interpreted in different ways *** diff --git a/epochX/cudacpp/gg_ttgg/src/mgOnGpuVectors.h b/epochX/cudacpp/gg_ttgg/src/mgOnGpuVectors.h index e1da365663..4e975b2ca1 100644 --- a/epochX/cudacpp/gg_ttgg/src/mgOnGpuVectors.h +++ b/epochX/cudacpp/gg_ttgg/src/mgOnGpuVectors.h @@ -13,9 +13,13 @@ namespace mgOnGpu { + #ifdef MGONGPU_CPPSIMD - const int neppV = neppM; + const int neppV = MGONGPU_CPPSIMD; + + // SANITY CHECK: cppAlign must be a multiple of neppV * sizeof(fptype) + static_assert( mgOnGpu::cppAlign % ( neppV * sizeof(fptype) ) == 0 ); // --- Type definition (using vector compiler extensions: need -march=...) #ifdef __clang__ // https://clang.llvm.org/docs/LanguageExtensions.html#vectors-and-extended-vectors @@ -96,7 +100,7 @@ namespace mgOnGpu #else // MGONGPU_CPPSIMD not defined - const int neppV = 1; // Note: also neppM is equal to 1 + const int neppV = 1; #endif } @@ -114,7 +118,7 @@ using mgOnGpu::bool_v; inline std::ostream& operator<<( std::ostream& out, const bool_v& v ) { out << "{ " << v[0]; - for ( int i=1; i #include @@ -22,7 +22,7 @@ namespace rambo2toNm0 // [NB: the output buffer includes both initial and final momenta, but only initial momenta are filled in] __global__ void getMomentaInitial( const fptype energy, // input: energy - fptype_sv momenta1d[] // output: momenta as AOSOA[npagM][npar][4][neppM] + fptype momenta1d[] // output: momenta as AOSOA[npagM][npar][4][neppM] #ifndef __CUDACC__ , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) #endif @@ -63,7 +63,7 @@ namespace rambo2toNm0 __global__ void getMomentaFinal( const fptype energy, // input: energy const fptype rnarray1d[], // input: random numbers in [0,1] as AOSOA[npagR][nparf][4][neppR] - fptype_sv momenta1d[], // output: momenta as AOSOA[npagM][npar][4][neppM] + fptype momenta1d[], // output: momenta as AOSOA[npagM][npar][4][neppM] fptype wgts[] // output: weights[nevt] #ifndef __CUDACC__ , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) diff --git a/epochX/cudacpp/gg_ttgg/src/rambo.h b/epochX/cudacpp/gg_ttgg/src/rambo.h index 030526759f..9d9e876fe3 100644 --- a/epochX/cudacpp/gg_ttgg/src/rambo.h +++ b/epochX/cudacpp/gg_ttgg/src/rambo.h @@ -3,7 +3,7 @@ #include "mgOnGpuConfig.h" #include "mgOnGpuTypes.h" -#include "mgOnGpuVectors.h" +//#include "mgOnGpuVectors.h" #include @@ -47,7 +47,7 @@ namespace rambo2toNm0 // [NB: the output buffer includes both initial and final momenta, but only initial momenta are filled in] __global__ void getMomentaInitial( const fptype energy, // input: energy - fptype_sv momenta1d[] // output: momenta as AOSOA[npagM][npar][4][neppM] + fptype momenta1d[] // output: momenta as AOSOA[npagM][npar][4][neppM] #ifndef __CUDACC__ , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) #endif @@ -60,7 +60,7 @@ namespace rambo2toNm0 __global__ void getMomentaFinal( const fptype energy, // input: energy const fptype rnarray1d[], // input: random numbers in [0,1] as AOSOA[npagR][nparf][4][neppR] - fptype_sv momenta1d[], // output: momenta as AOSOA[npagM][npar][4][neppM] + fptype momenta1d[], // output: momenta as AOSOA[npagM][npar][4][neppM] fptype wgts[] // output: weights[nevt] #ifndef __CUDACC__ , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) From 0de44180f3e8d6a670ec9c89a4f9de13e950df91 Mon Sep 17 00:00:00 2001 From: Andrea Valassi Date: Wed, 1 Dec 2021 11:55:52 +0100 Subject: [PATCH 23/34] [klas5X] in ggttgg manu, split the avxall target into five separate targets to allow parallel "make -j" builds --- epochX/cudacpp/gg_ttgg/SubProcesses/Makefile | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/epochX/cudacpp/gg_ttgg/SubProcesses/Makefile b/epochX/cudacpp/gg_ttgg/SubProcesses/Makefile index e57c3f2024..29cfc63217 100644 --- a/epochX/cudacpp/gg_ttgg/SubProcesses/Makefile +++ b/epochX/cudacpp/gg_ttgg/SubProcesses/Makefile @@ -330,18 +330,29 @@ $(GTESTLIBS): check: $(testmain) $(testmain) -avxall: +avxnone: @echo make USEBUILDDIR=1 AVX=none + +avxsse4: @echo make USEBUILDDIR=1 AVX=sse4 + +avxavx2: @echo make USEBUILDDIR=1 AVX=avx2 + +avx512y: @echo make USEBUILDDIR=1 AVX=512y + +avx512z: @echo make USEBUILDDIR=1 AVX=512z +# Split the avxall target into five separate targets to allow parallel "make -j" builds +avxall: avxnone avxsse4 avxavx2 avx512y avx512z + .PHONY: clean clean: From e67db4d995b5da5b5d39041ffeeff36cfa4d0e2e Mon Sep 17 00:00:00 2001 From: Andrea Valassi Date: Wed, 1 Dec 2021 12:49:30 +0100 Subject: [PATCH 24/34] [klas5X] in codegen, split avxall into five targets for parallel builds - regenerate 3 procs, sync eemumu manu --- .../madgraph/iolibs/template_files/gpu/Makefile | 13 ++++++++++++- .../ee_mumu.auto/CODEGEN_cudacpp_ee_mumu_log.txt | 8 ++++---- epochX/cudacpp/ee_mumu.auto/SubProcesses/Makefile | 13 ++++++++++++- epochX/cudacpp/ee_mumu/SubProcesses/Makefile | 13 ++++++++++++- .../gg_tt.auto/CODEGEN_cudacpp_gg_tt_log.txt | 8 ++++---- epochX/cudacpp/gg_tt.auto/SubProcesses/Makefile | 13 ++++++++++++- .../gg_ttgg.auto/CODEGEN_cudacpp_gg_ttgg_log.txt | 12 ++++++------ epochX/cudacpp/gg_ttgg.auto/SubProcesses/Makefile | 13 ++++++++++++- 8 files changed, 74 insertions(+), 19 deletions(-) diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/Makefile b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/Makefile index 102323ed4b..7f9dc9d506 100644 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/Makefile +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/Makefile @@ -330,18 +330,29 @@ $(GTESTLIBS): check: $(testmain) $(testmain) -avxall: +avxnone: @echo make USEBUILDDIR=1 AVX=none + +avxsse4: @echo make USEBUILDDIR=1 AVX=sse4 + +avxavx2: @echo make USEBUILDDIR=1 AVX=avx2 + +avx512y: @echo make USEBUILDDIR=1 AVX=512y + +avx512z: @echo make USEBUILDDIR=1 AVX=512z +# Split the avxall target into five separate targets to allow parallel "make -j" builds +avxall: avxnone avxsse4 avxavx2 avx512y avx512z + .PHONY: clean clean: diff --git a/epochX/cudacpp/ee_mumu.auto/CODEGEN_cudacpp_ee_mumu_log.txt b/epochX/cudacpp/ee_mumu.auto/CODEGEN_cudacpp_ee_mumu_log.txt index b3728dde47..75a0f96fd4 100644 --- a/epochX/cudacpp/ee_mumu.auto/CODEGEN_cudacpp_ee_mumu_log.txt +++ b/epochX/cudacpp/ee_mumu.auto/CODEGEN_cudacpp_ee_mumu_log.txt @@ -50,7 +50,7 @@ generate e+ e- > mu+ mu- No model currently active, so we import the Standard Model INFO: load particles INFO: load vertices -DEBUG: model prefixing takes 0.006864309310913086  +DEBUG: model prefixing takes 0.00683903694152832  INFO: Restrict model sm with file models/sm/restrict_default.dat . INFO: Change particles name to pass to MG5 convention DEBUG: True [misc.py at line 2192]  @@ -121,6 +121,6 @@ INFO: /data/avalassi/GPU2020/MG5aMC/2.7.0_gpu/CODEGEN_cudacpp_ee_mumu/src/. and DEBUG: Entering PLUGIN_ProcessExporter.finalize [output.py at line 154]  quit -real 0m4.641s -user 0m0.928s -sys 0m0.135s +real 0m4.192s +user 0m0.916s +sys 0m0.146s diff --git a/epochX/cudacpp/ee_mumu.auto/SubProcesses/Makefile b/epochX/cudacpp/ee_mumu.auto/SubProcesses/Makefile index e57c3f2024..29cfc63217 100644 --- a/epochX/cudacpp/ee_mumu.auto/SubProcesses/Makefile +++ b/epochX/cudacpp/ee_mumu.auto/SubProcesses/Makefile @@ -330,18 +330,29 @@ $(GTESTLIBS): check: $(testmain) $(testmain) -avxall: +avxnone: @echo make USEBUILDDIR=1 AVX=none + +avxsse4: @echo make USEBUILDDIR=1 AVX=sse4 + +avxavx2: @echo make USEBUILDDIR=1 AVX=avx2 + +avx512y: @echo make USEBUILDDIR=1 AVX=512y + +avx512z: @echo make USEBUILDDIR=1 AVX=512z +# Split the avxall target into five separate targets to allow parallel "make -j" builds +avxall: avxnone avxsse4 avxavx2 avx512y avx512z + .PHONY: clean clean: diff --git a/epochX/cudacpp/ee_mumu/SubProcesses/Makefile b/epochX/cudacpp/ee_mumu/SubProcesses/Makefile index e57c3f2024..29cfc63217 100644 --- a/epochX/cudacpp/ee_mumu/SubProcesses/Makefile +++ b/epochX/cudacpp/ee_mumu/SubProcesses/Makefile @@ -330,18 +330,29 @@ $(GTESTLIBS): check: $(testmain) $(testmain) -avxall: +avxnone: @echo make USEBUILDDIR=1 AVX=none + +avxsse4: @echo make USEBUILDDIR=1 AVX=sse4 + +avxavx2: @echo make USEBUILDDIR=1 AVX=avx2 + +avx512y: @echo make USEBUILDDIR=1 AVX=512y + +avx512z: @echo make USEBUILDDIR=1 AVX=512z +# Split the avxall target into five separate targets to allow parallel "make -j" builds +avxall: avxnone avxsse4 avxavx2 avx512y avx512z + .PHONY: clean clean: diff --git a/epochX/cudacpp/gg_tt.auto/CODEGEN_cudacpp_gg_tt_log.txt b/epochX/cudacpp/gg_tt.auto/CODEGEN_cudacpp_gg_tt_log.txt index 64e46f0fa7..23aca5f4a5 100644 --- a/epochX/cudacpp/gg_tt.auto/CODEGEN_cudacpp_gg_tt_log.txt +++ b/epochX/cudacpp/gg_tt.auto/CODEGEN_cudacpp_gg_tt_log.txt @@ -50,7 +50,7 @@ generate g g > t t~ No model currently active, so we import the Standard Model INFO: load particles INFO: load vertices -DEBUG: model prefixing takes 0.006934404373168945  +DEBUG: model prefixing takes 0.006844043731689453  INFO: Restrict model sm with file models/sm/restrict_default.dat . INFO: Change particles name to pass to MG5 convention DEBUG: True [misc.py at line 2192]  @@ -118,6 +118,6 @@ INFO: /data/avalassi/GPU2020/MG5aMC/2.7.0_gpu/CODEGEN_cudacpp_gg_tt/src/. and /d DEBUG: Entering PLUGIN_ProcessExporter.finalize [output.py at line 154]  quit -real 0m3.842s -user 0m0.806s -sys 0m0.116s +real 0m3.958s +user 0m0.792s +sys 0m0.129s diff --git a/epochX/cudacpp/gg_tt.auto/SubProcesses/Makefile b/epochX/cudacpp/gg_tt.auto/SubProcesses/Makefile index e57c3f2024..29cfc63217 100644 --- a/epochX/cudacpp/gg_tt.auto/SubProcesses/Makefile +++ b/epochX/cudacpp/gg_tt.auto/SubProcesses/Makefile @@ -330,18 +330,29 @@ $(GTESTLIBS): check: $(testmain) $(testmain) -avxall: +avxnone: @echo make USEBUILDDIR=1 AVX=none + +avxsse4: @echo make USEBUILDDIR=1 AVX=sse4 + +avxavx2: @echo make USEBUILDDIR=1 AVX=avx2 + +avx512y: @echo make USEBUILDDIR=1 AVX=512y + +avx512z: @echo make USEBUILDDIR=1 AVX=512z +# Split the avxall target into five separate targets to allow parallel "make -j" builds +avxall: avxnone avxsse4 avxavx2 avx512y avx512z + .PHONY: clean clean: diff --git a/epochX/cudacpp/gg_ttgg.auto/CODEGEN_cudacpp_gg_ttgg_log.txt b/epochX/cudacpp/gg_ttgg.auto/CODEGEN_cudacpp_gg_ttgg_log.txt index 8195fc5150..ae96ff7a46 100644 --- a/epochX/cudacpp/gg_ttgg.auto/CODEGEN_cudacpp_gg_ttgg_log.txt +++ b/epochX/cudacpp/gg_ttgg.auto/CODEGEN_cudacpp_gg_ttgg_log.txt @@ -50,7 +50,7 @@ generate g g > t t~ g g No model currently active, so we import the Standard Model INFO: load particles INFO: load vertices -DEBUG: model prefixing takes 0.006870269775390625  +DEBUG: model prefixing takes 0.006913185119628906  INFO: Restrict model sm with file models/sm/restrict_default.dat . INFO: Change particles name to pass to MG5 convention DEBUG: True [misc.py at line 2192]  @@ -67,7 +67,7 @@ INFO: Please specify coupling orders to bypass this step. INFO: Trying coupling order WEIGHTED<=4: WEIGTHED IS QCD+2*QED INFO: Trying process: g g > t t~ g g WEIGHTED<=4 @1 INFO: Process has 123 diagrams -1 processes with 123 diagrams generated in 0.207 s +1 processes with 123 diagrams generated in 0.209 s Total: 1 processes with 123 diagrams output standalone_cudacpp CODEGEN_cudacpp_gg_ttgg Output will be done with PLUGIN: CUDACPP_SA_OUTPUT @@ -100,7 +100,7 @@ INFO: Created files CPPProcess.h and CPPProcess.cc in directory /data/avalassi/G DEBUG: Entering PLUGIN_OneProcessExporter.edit_check_sa [model_handling.py at line 907]  DEBUG: Entering PLUGIN_OneProcessExporter.edit_mgonGPU [model_handling.py at line 922]  DEBUG: Entering PLUGIN_OneProcessExporter.edit_processidfile [model_handling.py at line 929]  -Generated helas calls for 1 subprocesses (123 diagrams) in 0.558 s +Generated helas calls for 1 subprocesses (123 diagrams) in 0.557 s DEBUG: Entering PLUGIN_ProcessExporter.convert_model (create the model) [output.py at line 145]  ALOHA: aloha creates routines (starting by VVV1) VVV1 @@ -138,6 +138,6 @@ INFO: /data/avalassi/GPU2020/MG5aMC/2.7.0_gpu/CODEGEN_cudacpp_gg_ttgg/src/. and DEBUG: Entering PLUGIN_ProcessExporter.finalize [output.py at line 154]  quit -real 0m5.019s -user 0m1.967s -sys 0m0.149s +real 0m5.154s +user 0m1.994s +sys 0m0.138s diff --git a/epochX/cudacpp/gg_ttgg.auto/SubProcesses/Makefile b/epochX/cudacpp/gg_ttgg.auto/SubProcesses/Makefile index e57c3f2024..29cfc63217 100644 --- a/epochX/cudacpp/gg_ttgg.auto/SubProcesses/Makefile +++ b/epochX/cudacpp/gg_ttgg.auto/SubProcesses/Makefile @@ -330,18 +330,29 @@ $(GTESTLIBS): check: $(testmain) $(testmain) -avxall: +avxnone: @echo make USEBUILDDIR=1 AVX=none + +avxsse4: @echo make USEBUILDDIR=1 AVX=sse4 + +avxavx2: @echo make USEBUILDDIR=1 AVX=avx2 + +avx512y: @echo make USEBUILDDIR=1 AVX=512y + +avx512z: @echo make USEBUILDDIR=1 AVX=512z +# Split the avxall target into five separate targets to allow parallel "make -j" builds +avxall: avxnone avxsse4 avxavx2 avx512y avx512z + .PHONY: clean clean: From 5bdcd36776ec1dbd45d36b866d24fdce33222781 Mon Sep 17 00:00:00 2001 From: Andrea Valassi Date: Wed, 1 Dec 2021 13:05:07 +0100 Subject: [PATCH 25/34] [klas5X] (ALMOST COMPLETE KLAS5X) simplify -avxall in throughputX script and add -makej option to enable make -j --- epochX/cudacpp/tput/teeThroughputX.sh | 2 +- epochX/cudacpp/tput/throughputX.sh | 17 ++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/epochX/cudacpp/tput/teeThroughputX.sh b/epochX/cudacpp/tput/teeThroughputX.sh index f5be40d895..7ce57c36ae 100755 --- a/epochX/cudacpp/tput/teeThroughputX.sh +++ b/epochX/cudacpp/tput/teeThroughputX.sh @@ -95,7 +95,7 @@ for step in $steps; do printf "\n%80s\n" |tr " " "*" printf "*** ./throughputX.sh -makeonly $args" printf "\n%80s\n" |tr " " "*" - if ! ./throughputX.sh -makeonly $args; then exit 1; fi + if ! ./throughputX.sh -makeonly -makej $args; then exit 1; fi else logfile=logs_${proc#-}_${suff}/log_${proc#-}_${suff}_${fptype}_inl${helinl}.txt printf "\n%80s\n" |tr " " "*" diff --git a/epochX/cudacpp/tput/throughputX.sh b/epochX/cudacpp/tput/throughputX.sh index b966516931..2cf3efb740 100755 --- a/epochX/cudacpp/tput/throughputX.sh +++ b/epochX/cudacpp/tput/throughputX.sh @@ -17,13 +17,14 @@ fptypes="d" helinls="0" suffs="/" maketype= +makej= detailed=0 gtest=0 verbose=0 function usage() { - echo "Usage: $0 [-nocpp|[-omp][-avxall][-nocuda]] [-3a3b] [-div] [-req] [-flt|-fltonly] [-inl|-inlonly] [-auto|-autoonly] [-makeonly|-makeclean|-makecleanonly] [-detailed] [-gtest] [-v]" + echo "Usage: $0 [-nocpp|[-omp][-avxall][-nocuda]] [-3a3b] [-div] [-req] [-flt|-fltonly] [-inl|-inlonly] [-auto|-autoonly] [-makeonly|-makeclean|-makecleanonly] [-makej] [-detailed] [-gtest] [-v]" exit 1 } @@ -101,6 +102,9 @@ while [ "$1" != "" ]; do fi maketype="$1" shift + elif [ "$1" == "-makej" ]; then + makej=-j + shift elif [ "$1" == "-detailed" ]; then detailed=1 shift @@ -205,12 +209,11 @@ for suff in $suffs; do export HELINL=$helinl for fptype in $fptypes; do export FPTYPE=$fptype - make -j AVX=none; echo - if [ "${avxall}" == "1" ]; then make -j AVX=sse4; echo; fi - if [ "${avxall}" == "1" ]; then make -j AVX=avx2; echo; fi - if [ "$(grep -m1 -c avx512vl /proc/cpuinfo)" == "1" ]; then # skip avx512 if not supported! - if [ "${cpp}" == "1" ]; then make -j AVX=512y; echo; fi # use 512y as C++ ref even if avx2 is faster on clang - if [ "${avxall}" == "1" ]; then make -j AVX=512z; echo; fi + if [ "${avxall}" == "1" ]; then + make ${makej} avxall; echo + else + make ${makej} AVX=none; echo + make ${makej} AVX=512y; echo fi done done From cb7aa602b1a392ed75f057a7e8400a52ff6b66e1 Mon Sep 17 00:00:00 2001 From: Andrea Valassi Date: Wed, 1 Dec 2021 21:01:44 +0100 Subject: [PATCH 26/34] [klas5X] port to epochX/klas5X the latest few changes in epoch1/klas5 eemumu --- epochX/cudacpp/ee_mumu/SubProcesses/Memory.h | 4 ++++ .../SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc | 5 ++++- epochX/cudacpp/ee_mumu/src/mgOnGpuConfig.h | 7 ++++--- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/epochX/cudacpp/ee_mumu/SubProcesses/Memory.h b/epochX/cudacpp/ee_mumu/SubProcesses/Memory.h index a9c08d3dd0..34b5e02071 100644 --- a/epochX/cudacpp/ee_mumu/SubProcesses/Memory.h +++ b/epochX/cudacpp/ee_mumu/SubProcesses/Memory.h @@ -51,6 +51,7 @@ template struct CppHstDeleter { void operator()(T* mem) { ::operator delete( mem, std::align_val_t{ mgOnGpu::cppAlign } ); + //::operator delete( mem-1, std::align_val_t{ mgOnGpu::cppAlign } ); // TEST MISALIGNMENT! } }; @@ -58,8 +59,10 @@ template inline std::unique_ptr> hstMakeUnique(std::size_t N) { // See https://www.bfilipek.com/2019/08/newnew-align.html#requesting-different-alignment return std::unique_ptr>{ new( std::align_val_t{ mgOnGpu::cppAlign } ) T[N]() }; + //return std::unique_ptr>{ new( std::align_val_t{ mgOnGpu::cppAlign } ) T[N+1]() + 1 }; // TEST MISALIGNMENT! }; +/* #ifdef MGONGPU_CPPSIMD template<> inline @@ -68,6 +71,7 @@ std::unique_ptr> hstMakeUnique(std::size_t N }; #endif +*/ #endif diff --git a/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc b/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc index d49bf181ce..ece5097c11 100644 --- a/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc +++ b/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc @@ -406,7 +406,7 @@ namespace Proc , const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) { assert( (size_t)(allmomenta) % mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment - //assert( (size_t)(allMEs) % mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment + assert( (size_t)(allMEs) % mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment const int maxtry0 = ( neppV > 16 ? neppV : 16 ); // 16, but at least neppV (otherwise the npagV loop does not even start) fptype allMEsLast[maxtry0] = { 0 }; const int maxtry = std::min( maxtry0, nevt ); // 16, but at most nevt (avoid invalid memory access if nevt Date: Wed, 1 Dec 2021 21:52:05 +0100 Subject: [PATCH 27/34] [klas5X] enable make -j only if an explicit -makej is given in the teeThroughput script --- epochX/cudacpp/tput/teeThroughputX.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/epochX/cudacpp/tput/teeThroughputX.sh b/epochX/cudacpp/tput/teeThroughputX.sh index 7ce57c36ae..1188af02cb 100755 --- a/epochX/cudacpp/tput/teeThroughputX.sh +++ b/epochX/cudacpp/tput/teeThroughputX.sh @@ -4,7 +4,7 @@ cd $(dirname $0) function usage() { - echo "Usage: $0 [-auto|-autoonly] [-flt|-fltonly] [-inl|-inlonly] [-makeonly] [-makeclean]" + echo "Usage: $0 [-auto|-autoonly] [-flt|-fltonly] [-inl|-inlonly] [-makeonly] [-makeclean] [-makej]" exit 1 } @@ -16,6 +16,7 @@ suffs="manu" fptypes="d" helinls="0" steps="make test" +makej= for arg in $*; do if [ "$arg" == "-eemumu" ]; then if [ "$eemumu" == "" ]; then procs+=${procs:+ }${arg}; fi @@ -56,6 +57,9 @@ for arg in $*; do elif [ "${steps}" == "make" ]; then steps="makeclean make" fi + elif [ "$arg" == "-makej" ]; then + makej=-makej + shift else echo "ERROR! Invalid option '$arg'"; usage fi @@ -95,7 +99,7 @@ for step in $steps; do printf "\n%80s\n" |tr " " "*" printf "*** ./throughputX.sh -makeonly $args" printf "\n%80s\n" |tr " " "*" - if ! ./throughputX.sh -makeonly -makej $args; then exit 1; fi + if ! ./throughputX.sh -makeonly ${makej} $args; then exit 1; fi else logfile=logs_${proc#-}_${suff}/log_${proc#-}_${suff}_${fptype}_inl${helinl}.txt printf "\n%80s\n" |tr " " "*" From b2abb48fef07ad51496a76556f4770092fce0ded Mon Sep 17 00:00:00 2001 From: Andrea Valassi Date: Wed, 1 Dec 2021 22:00:42 +0100 Subject: [PATCH 28/34] [klas5X] rerun eemumu manual tests - a bit slower after adding one sanity check? --- .../log_eemumu_manu_d_inl0.txt | 106 +++++++++++------- .../log_eemumu_manu_d_inl1.txt | 106 +++++++++++------- .../log_eemumu_manu_f_inl0.txt | 106 +++++++++++------- .../log_eemumu_manu_f_inl1.txt | 106 +++++++++++------- 4 files changed, 256 insertions(+), 168 deletions(-) diff --git a/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_d_inl0.txt b/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_d_inl0.txt index 24715a4b4e..71d8f86e70 100644 --- a/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_d_inl0.txt +++ b/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_d_inl0.txt @@ -1,59 +1,81 @@ /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum OMPFLAGS= +AVX=512y +FPTYPE=d +HELINL=0 +RNDGEN=curdev +Building in BUILDDIR=build.512y_d_inl0 for tag=512y_d_inl0_curdev (USEBUILDDIR is set = 1) + +make USEBUILDDIR=1 AVX=none +OMPFLAGS= AVX=none FPTYPE=d HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.none_d_inl0 for tag=none_d_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.none_d_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.none_d_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=sse4 OMPFLAGS= AVX=sse4 FPTYPE=d HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.sse4_d_inl0 for tag=sse4_d_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.sse4_d_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.sse4_d_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=avx2 OMPFLAGS= AVX=avx2 FPTYPE=d HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.avx2_d_inl0 for tag=avx2_d_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.avx2_d_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.avx2_d_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=512y OMPFLAGS= AVX=512y FPTYPE=d HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.512y_d_inl0 for tag=512y_d_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512y_d_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.512y_d_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=512z OMPFLAGS= AVX=512z FPTYPE=d HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.512z_d_inl0 for tag=512z_d_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512z_d_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.512z_d_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' -DATE: 2021-12-01_10:07:10 +DATE: 2021-12-01_21:58:04 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl0/gcheck.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 6.534485e+08 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.116555e+09 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 5.957319e+08 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.110528e+09 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 0.709466 sec - 371,909,202 cycles:u # 0.399 GHz - 697,385,836 instructions:u # 1.88 insn per cycle - 0.996526017 seconds time elapsed +TOTAL : 0.732748 sec + 396,994,334 cycles:u # 0.412 GHz + 739,059,272 instructions:u # 1.86 insn per cycle + 1.028274926 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 130 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... @@ -62,13 +84,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.337315e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.337315e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.354112e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.354112e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 7.047599 sec - 18,666,706,719 cycles:u # 2.647 GHz - 48,224,158,012 instructions:u # 2.58 insn per cycle - 7.055088398 seconds time elapsed +TOTAL : 6.991839 sec + 18,522,301,845 cycles:u # 2.648 GHz + 48,224,155,362 instructions:u # 2.60 insn per cycle + 7.000208392 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 621) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl0/runTest.exe @@ -78,13 +100,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[2] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 2.565767e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 2.565767e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 2.564081e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 2.564081e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 4.823406 sec - 12,729,876,419 cycles:u # 2.637 GHz - 29,850,563,434 instructions:u # 2.34 insn per cycle - 4.831352437 seconds time elapsed +TOTAL : 4.836123 sec + 12,759,577,192 cycles:u # 2.636 GHz + 29,850,564,087 instructions:u # 2.34 insn per cycle + 4.844162095 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 3297) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_d_inl0/runTest.exe @@ -94,13 +116,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 4.556887e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.556887e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.565133e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.565133e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 3.663924 sec - 9,087,880,283 cycles:u # 2.478 GHz - 16,644,667,763 instructions:u # 1.83 insn per cycle - 3.671637372 seconds time elapsed +TOTAL : 3.673703 sec + 9,115,434,244 cycles:u # 2.478 GHz + 16,644,668,367 instructions:u # 1.83 insn per cycle + 3.681693279 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2831) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_d_inl0/runTest.exe @@ -110,13 +132,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 4.921532e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.921532e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.909323e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.909323e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 3.602738 sec - 8,968,063,737 cycles:u # 2.486 GHz - 16,531,423,183 instructions:u # 1.84 insn per cycle - 3.610374997 seconds time elapsed +TOTAL : 3.594048 sec + 8,942,444,328 cycles:u # 2.485 GHz + 16,531,423,242 instructions:u # 1.85 insn per cycle + 3.602091761 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2728) (512y: 52) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_d_inl0/runTest.exe @@ -126,13 +148,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 3.828049e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.828049e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 3.676735e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.676735e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 3.958997 sec - 8,638,325,570 cycles:u # 2.180 GHz - 13,280,272,453 instructions:u # 1.54 insn per cycle - 3.966662948 seconds time elapsed +TOTAL : 4.049430 sec + 8,791,223,746 cycles:u # 2.169 GHz + 13,280,272,515 instructions:u # 1.51 insn per cycle + 4.057343967 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1319) (512y: 66) (512z: 2147) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_d_inl0/runTest.exe diff --git a/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_d_inl1.txt b/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_d_inl1.txt index 17c1e59cc3..e453d683c2 100644 --- a/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_d_inl1.txt +++ b/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_d_inl1.txt @@ -1,59 +1,81 @@ /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum OMPFLAGS= +AVX=512y +FPTYPE=d +HELINL=1 +RNDGEN=curdev +Building in BUILDDIR=build.512y_d_inl1 for tag=512y_d_inl1_curdev (USEBUILDDIR is set = 1) + +make USEBUILDDIR=1 AVX=none +OMPFLAGS= AVX=none FPTYPE=d HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.none_d_inl1 for tag=none_d_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.none_d_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.none_d_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=sse4 OMPFLAGS= AVX=sse4 FPTYPE=d HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.sse4_d_inl1 for tag=sse4_d_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.sse4_d_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.sse4_d_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=avx2 OMPFLAGS= AVX=avx2 FPTYPE=d HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.avx2_d_inl1 for tag=avx2_d_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.avx2_d_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.avx2_d_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=512y OMPFLAGS= AVX=512y FPTYPE=d HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.512y_d_inl1 for tag=512y_d_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512y_d_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.512y_d_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=512z OMPFLAGS= AVX=512z FPTYPE=d HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.512z_d_inl1 for tag=512z_d_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512z_d_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.512z_d_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' -DATE: 2021-12-01_10:07:46 +DATE: 2021-12-01_21:58:39 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl1/gcheck.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 6.494723e+08 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.109138e+09 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 5.919373e+08 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.109577e+09 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 1.226259 sec - 997,952,337 cycles:u # 0.839 GHz - 1,981,968,331 instructions:u # 1.99 insn per cycle - 1.518449844 seconds time elapsed +TOTAL : 0.729794 sec + 394,330,255 cycles:u # 0.410 GHz + 734,519,727 instructions:u # 1.86 insn per cycle + 1.026025161 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 130 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... @@ -62,13 +84,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 4.736845e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.736845e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.726360e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.726360e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 3.677158 sec - 9,670,909,975 cycles:u # 2.627 GHz - 17,889,570,187 instructions:u # 1.85 insn per cycle - 3.684775451 seconds time elapsed +TOTAL : 3.668891 sec + 9,654,425,372 cycles:u # 2.628 GHz + 17,889,569,619 instructions:u # 1.85 insn per cycle + 3.677068240 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 2595) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl1/runTest.exe @@ -78,13 +100,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[2] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 9.455274e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 9.455274e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 9.381860e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 9.381860e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 3.046095 sec - 7,995,654,626 cycles:u # 2.622 GHz - 13,964,472,664 instructions:u # 1.75 insn per cycle - 3.053929213 seconds time elapsed +TOTAL : 3.052269 sec + 8,011,338,634 cycles:u # 2.620 GHz + 13,964,473,137 instructions:u # 1.74 insn per cycle + 3.060276458 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 432) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_d_inl1/runTest.exe @@ -94,13 +116,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.303904e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.303904e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.293550e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.293550e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 2.773556 sec - 7,073,285,248 cycles:u # 2.547 GHz - 11,086,106,492 instructions:u # 1.57 insn per cycle - 2.781104378 seconds time elapsed +TOTAL : 2.784430 sec + 7,096,839,537 cycles:u # 2.544 GHz + 11,086,106,832 instructions:u # 1.56 insn per cycle + 2.792751894 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 461) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_d_inl1/runTest.exe @@ -110,13 +132,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.383099e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.383099e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.359076e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.359076e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 2.767352 sec - 7,074,744,001 cycles:u # 2.552 GHz - 10,919,382,397 instructions:u # 1.54 insn per cycle - 2.775205710 seconds time elapsed +TOTAL : 2.776058 sec + 7,083,525,857 cycles:u # 2.548 GHz + 10,919,382,570 instructions:u # 1.54 insn per cycle + 2.784045302 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 412) (512y: 1) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_d_inl1/runTest.exe @@ -126,13 +148,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 9.344111e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 9.344111e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 9.087523e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 9.087523e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 3.015544 sec - 7,170,726,069 cycles:u # 2.377 GHz - 10,478,969,947 instructions:u # 1.46 insn per cycle - 3.023095729 seconds time elapsed +TOTAL : 3.037580 sec + 7,216,761,782 cycles:u # 2.372 GHz + 10,478,969,816 instructions:u # 1.45 insn per cycle + 3.045604626 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 193) (512y: 2) (512z: 230) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_d_inl1/runTest.exe diff --git a/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_f_inl0.txt b/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_f_inl0.txt index 30e3d856fb..e61869d016 100644 --- a/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_f_inl0.txt +++ b/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_f_inl0.txt @@ -1,59 +1,81 @@ /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum OMPFLAGS= +AVX=512y +FPTYPE=f +HELINL=0 +RNDGEN=curdev +Building in BUILDDIR=build.512y_f_inl0 for tag=512y_f_inl0_curdev (USEBUILDDIR is set = 1) + +make USEBUILDDIR=1 AVX=none +OMPFLAGS= AVX=none FPTYPE=f HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.none_f_inl0 for tag=none_f_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.none_f_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.none_f_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=sse4 OMPFLAGS= AVX=sse4 FPTYPE=f HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.sse4_f_inl0 for tag=sse4_f_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.sse4_f_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.sse4_f_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=avx2 OMPFLAGS= AVX=avx2 FPTYPE=f HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.avx2_f_inl0 for tag=avx2_f_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.avx2_f_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.avx2_f_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=512y OMPFLAGS= AVX=512y FPTYPE=f HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.512y_f_inl0 for tag=512y_f_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512y_f_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.512y_f_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=512z OMPFLAGS= AVX=512z FPTYPE=f HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.512z_f_inl0 for tag=512z_f_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512z_f_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.512z_f_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' -DATE: 2021-12-01_10:08:12 +DATE: 2021-12-01_21:59:06 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl0/gcheck.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=2, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 1.567154e+09 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.236935e+09 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.427625e+09 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.256233e+09 ) sec^-1 MeanMatrixElemValue = ( 1.371686e-02 +- 3.270219e-06 ) GeV^0 -TOTAL : 0.842427 sec - 716,989,672 cycles:u # 0.704 GHz - 1,484,422,831 instructions:u # 2.07 insn per cycle - 1.126361376 seconds time elapsed +TOTAL : 0.669291 sec + 340,663,519 cycles:u # 0.384 GHz + 672,342,639 instructions:u # 1.97 insn per cycle + 0.967024151 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 56 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... @@ -62,13 +84,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=6, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.224589e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.224589e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.227750e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.227750e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371707e-02 +- 3.270376e-06 ) GeV^0 -TOTAL : 7.053663 sec - 18,764,093,689 cycles:u # 2.659 GHz - 47,311,642,026 instructions:u # 2.52 insn per cycle - 7.060851302 seconds time elapsed +TOTAL : 7.045155 sec + 18,729,859,154 cycles:u # 2.658 GHz + 47,311,640,742 instructions:u # 2.53 insn per cycle + 7.052949347 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 559) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl0/runTest.exe @@ -78,13 +100,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=6, zero=0) Internal loops fptype_sv = VECTOR[4] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 4.573579e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.573579e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.583214e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.583214e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270375e-06 ) GeV^0 -TOTAL : 3.344023 sec - 8,846,286,898 cycles:u # 2.642 GHz - 19,615,196,579 instructions:u # 2.22 insn per cycle - 3.351351467 seconds time elapsed +TOTAL : 3.370620 sec + 8,915,711,214 cycles:u # 2.642 GHz + 19,615,194,859 instructions:u # 2.20 insn per cycle + 3.378548883 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 3969) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_f_inl0/runTest.exe @@ -94,13 +116,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=5, zero=0) Internal loops fptype_sv = VECTOR[8] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 8.181232e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 8.181232e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 8.183244e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 8.183244e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371705e-02 +- 3.270339e-06 ) GeV^0 -TOTAL : 2.698787 sec - 6,823,046,265 cycles:u # 2.524 GHz - 12,524,650,277 instructions:u # 1.84 insn per cycle - 2.705979783 seconds time elapsed +TOTAL : 2.684441 sec + 6,783,050,356 cycles:u # 2.522 GHz + 12,524,650,491 instructions:u # 1.85 insn per cycle + 2.692018765 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 3175) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_f_inl0/runTest.exe @@ -110,13 +132,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=5, zero=0) Internal loops fptype_sv = VECTOR[8] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 8.961058e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 8.961058e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 8.980040e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 8.980040e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371705e-02 +- 3.270339e-06 ) GeV^0 -TOTAL : 2.587116 sec - 6,548,070,094 cycles:u # 2.527 GHz - 12,477,465,603 instructions:u # 1.91 insn per cycle - 2.594729483 seconds time elapsed +TOTAL : 2.606645 sec + 6,599,980,765 cycles:u # 2.527 GHz + 12,477,467,361 instructions:u # 1.89 insn per cycle + 2.614446054 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 3069) (512y: 26) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_f_inl0/runTest.exe @@ -126,13 +148,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=5, zero=0) Internal loops fptype_sv = VECTOR[16] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 7.493309e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 7.493309e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 7.346633e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 7.346633e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371705e-02 +- 3.270340e-06 ) GeV^0 -TOTAL : 2.727509 sec - 6,281,035,658 cycles:u # 2.300 GHz - 10,798,421,531 instructions:u # 1.72 insn per cycle - 2.734747652 seconds time elapsed +TOTAL : 2.745844 sec + 6,303,361,200 cycles:u # 2.291 GHz + 10,798,421,576 instructions:u # 1.71 insn per cycle + 2.753608906 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1772) (512y: 12) (512z: 2256) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_f_inl0/runTest.exe diff --git a/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_f_inl1.txt b/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_f_inl1.txt index adffd368b3..e405135d2a 100644 --- a/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_f_inl1.txt +++ b/epochX/cudacpp/tput/logs_eemumu_manu/log_eemumu_manu_f_inl1.txt @@ -1,59 +1,81 @@ /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum OMPFLAGS= +AVX=512y +FPTYPE=f +HELINL=1 +RNDGEN=curdev +Building in BUILDDIR=build.512y_f_inl1 for tag=512y_f_inl1_curdev (USEBUILDDIR is set = 1) + +make USEBUILDDIR=1 AVX=none +OMPFLAGS= AVX=none FPTYPE=f HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.none_f_inl1 for tag=none_f_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.none_f_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.none_f_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=sse4 OMPFLAGS= AVX=sse4 FPTYPE=f HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.sse4_f_inl1 for tag=sse4_f_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.sse4_f_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.sse4_f_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=avx2 OMPFLAGS= AVX=avx2 FPTYPE=f HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.avx2_f_inl1 for tag=avx2_f_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.avx2_f_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.avx2_f_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=512y OMPFLAGS= AVX=512y FPTYPE=f HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.512y_f_inl1 for tag=512y_f_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512y_f_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.512y_f_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=512z OMPFLAGS= AVX=512z FPTYPE=f HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.512z_f_inl1 for tag=512z_f_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512z_f_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.512z_f_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' -DATE: 2021-12-01_10:08:43 +DATE: 2021-12-01_21:59:36 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl1/gcheck.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=2, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 1.577348e+09 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.256680e+09 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.415544e+09 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.241284e+09 ) sec^-1 MeanMatrixElemValue = ( 1.371686e-02 +- 3.270219e-06 ) GeV^0 -TOTAL : 0.907275 sec - 718,084,580 cycles:u # 0.706 GHz - 1,482,928,433 instructions:u # 2.07 insn per cycle - 1.190328356 seconds time elapsed +TOTAL : 0.653274 sec + 338,494,738 cycles:u # 0.386 GHz + 664,838,163 instructions:u # 1.96 insn per cycle + 0.939786552 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 56 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... @@ -62,13 +84,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=6, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.203281e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.203281e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.200711e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.200711e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270375e-06 ) GeV^0 -TOTAL : 2.431255 sec - 6,417,013,841 cycles:u # 2.634 GHz - 12,595,066,654 instructions:u # 1.96 insn per cycle - 2.438826820 seconds time elapsed +TOTAL : 2.443712 sec + 6,450,825,445 cycles:u # 2.634 GHz + 12,595,066,498 instructions:u # 1.95 insn per cycle + 2.451707004 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 796) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl1/runTest.exe @@ -78,13 +100,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=6, zero=0) Internal loops fptype_sv = VECTOR[4] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.923450e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.923450e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.905316e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.905316e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270375e-06 ) GeV^0 -TOTAL : 2.297739 sec - 6,058,657,245 cycles:u # 2.632 GHz - 11,389,032,042 instructions:u # 1.88 insn per cycle - 2.304986309 seconds time elapsed +TOTAL : 2.306185 sec + 6,073,875,427 cycles:u # 2.628 GHz + 11,389,030,202 instructions:u # 1.88 insn per cycle + 2.313780178 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 508) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_f_inl1/runTest.exe @@ -94,13 +116,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=5, zero=0) Internal loops fptype_sv = VECTOR[8] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 2.894889e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 2.894889e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 2.883390e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 2.883390e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371705e-02 +- 3.270339e-06 ) GeV^0 -TOTAL : 2.105405 sec - 5,454,622,456 cycles:u # 2.585 GHz - 9,655,713,280 instructions:u # 1.77 insn per cycle - 2.113229620 seconds time elapsed +TOTAL : 2.128015 sec + 5,517,380,840 cycles:u # 2.587 GHz + 9,655,713,434 instructions:u # 1.75 insn per cycle + 2.136040937 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 533) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_f_inl1/runTest.exe @@ -110,13 +132,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=5, zero=0) Internal loops fptype_sv = VECTOR[8] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 3.028386e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.028386e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 2.938130e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 2.938130e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371705e-02 +- 3.270339e-06 ) GeV^0 -TOTAL : 2.093570 sec - 5,428,962,159 cycles:u # 2.588 GHz - 9,583,362,532 instructions:u # 1.77 insn per cycle - 2.101081895 seconds time elapsed +TOTAL : 2.123205 sec + 5,504,030,300 cycles:u # 2.587 GHz + 9,583,364,167 instructions:u # 1.74 insn per cycle + 2.130848906 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 497) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_f_inl1/runTest.exe @@ -126,13 +148,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=5, zero=0) Internal loops fptype_sv = VECTOR[16] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 2.059238e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 2.059238e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.982221e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.982221e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371705e-02 +- 3.270340e-06 ) GeV^0 -TOTAL : 2.192812 sec - 5,434,991,260 cycles:u # 2.474 GHz - 9,332,494,253 instructions:u # 1.72 insn per cycle - 2.200097462 seconds time elapsed +TOTAL : 2.204938 sec + 5,457,154,323 cycles:u # 2.470 GHz + 9,332,494,167 instructions:u # 1.71 insn per cycle + 2.212867763 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 293) (512y: 0) (512z: 251) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_f_inl1/runTest.exe From 34e1a95ae276e954296325cc356bb95ae9a957e6 Mon Sep 17 00:00:00 2001 From: Andrea Valassi Date: Wed, 1 Dec 2021 22:08:04 +0100 Subject: [PATCH 29/34] [klas5X] backport latest eemumu manu changes t codegen, regenerate eemumu auto --- .../madgraph/iolibs/template_files/gpu/Memory.h | 4 ++++ .../madgraph/iolibs/template_files/gpu/mgOnGpuConfig.h | 7 ++++--- .../template_files/gpu/process_function_definitions.inc | 5 ++++- .../cudacpp/ee_mumu.auto/CODEGEN_cudacpp_ee_mumu_log.txt | 8 ++++---- epochX/cudacpp/ee_mumu.auto/SubProcesses/Memory.h | 4 ++++ .../SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc | 5 ++++- epochX/cudacpp/ee_mumu.auto/src/mgOnGpuConfig.h | 7 ++++--- 7 files changed, 28 insertions(+), 12 deletions(-) diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/Memory.h b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/Memory.h index a9c08d3dd0..34b5e02071 100644 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/Memory.h +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/Memory.h @@ -51,6 +51,7 @@ template struct CppHstDeleter { void operator()(T* mem) { ::operator delete( mem, std::align_val_t{ mgOnGpu::cppAlign } ); + //::operator delete( mem-1, std::align_val_t{ mgOnGpu::cppAlign } ); // TEST MISALIGNMENT! } }; @@ -58,8 +59,10 @@ template inline std::unique_ptr> hstMakeUnique(std::size_t N) { // See https://www.bfilipek.com/2019/08/newnew-align.html#requesting-different-alignment return std::unique_ptr>{ new( std::align_val_t{ mgOnGpu::cppAlign } ) T[N]() }; + //return std::unique_ptr>{ new( std::align_val_t{ mgOnGpu::cppAlign } ) T[N+1]() + 1 }; // TEST MISALIGNMENT! }; +/* #ifdef MGONGPU_CPPSIMD template<> inline @@ -68,6 +71,7 @@ std::unique_ptr> hstMakeUnique(std::size_t N }; #endif +*/ #endif diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/mgOnGpuConfig.h b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/mgOnGpuConfig.h index 23e45ca5b1..09de460b9b 100644 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/mgOnGpuConfig.h +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/mgOnGpuConfig.h @@ -161,12 +161,13 @@ namespace mgOnGpu // ----------------------------------------------------------------------------------------------- #ifdef MGONGPU_CPPSIMD const int neppM = MGONGPU_CPPSIMD; // (DEFAULT) neppM=neppV for optimal performance -#else - const int neppM = 1; // (DEFAULT) neppM=neppV for optimal performance (NB: this is equivalent to AOS) -#endif //const int neppM = 64/sizeof(fptype); // maximum CPU vector width (512 bits): 8 (DOUBLE) or 16 (FLOAT) //const int neppM = 32/sizeof(fptype); // lower CPU vector width (256 bits): 4 (DOUBLE) or 8 (FLOAT) //const int neppM = 1; // *** NB: this is equivalent to AOS *** + //const int neppM = MGONGPU_CPPSIMD*2; // FOR TESTS +#else + const int neppM = 1; // (DEFAULT) neppM=neppV for optimal performance (NB: this is equivalent to AOS) +#endif #endif // Number of Events Per Page in the random number AOSOA memory layout diff --git a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/process_function_definitions.inc b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/process_function_definitions.inc index 0badaa4bfe..24acd5bf13 100644 --- a/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/process_function_definitions.inc +++ b/epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/process_function_definitions.inc @@ -202,7 +202,7 @@ namespace Proc , const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) { assert( (size_t)(allmomenta) %% mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment - //assert( (size_t)(allMEs) %% mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment + assert( (size_t)(allMEs) %% mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment const int maxtry0 = ( neppV > 16 ? neppV : 16 ); // 16, but at least neppV (otherwise the npagV loop does not even start) fptype allMEsLast[maxtry0] = { 0 }; const int maxtry = std::min( maxtry0, nevt ); // 16, but at most nevt (avoid invalid memory access if nevt mu+ mu- No model currently active, so we import the Standard Model INFO: load particles INFO: load vertices -DEBUG: model prefixing takes 0.00683903694152832  +DEBUG: model prefixing takes 0.00686192512512207  INFO: Restrict model sm with file models/sm/restrict_default.dat . INFO: Change particles name to pass to MG5 convention DEBUG: True [misc.py at line 2192]  @@ -121,6 +121,6 @@ INFO: /data/avalassi/GPU2020/MG5aMC/2.7.0_gpu/CODEGEN_cudacpp_ee_mumu/src/. and DEBUG: Entering PLUGIN_ProcessExporter.finalize [output.py at line 154]  quit -real 0m4.192s -user 0m0.916s -sys 0m0.146s +real 0m3.871s +user 0m0.928s +sys 0m0.140s diff --git a/epochX/cudacpp/ee_mumu.auto/SubProcesses/Memory.h b/epochX/cudacpp/ee_mumu.auto/SubProcesses/Memory.h index a9c08d3dd0..34b5e02071 100644 --- a/epochX/cudacpp/ee_mumu.auto/SubProcesses/Memory.h +++ b/epochX/cudacpp/ee_mumu.auto/SubProcesses/Memory.h @@ -51,6 +51,7 @@ template struct CppHstDeleter { void operator()(T* mem) { ::operator delete( mem, std::align_val_t{ mgOnGpu::cppAlign } ); + //::operator delete( mem-1, std::align_val_t{ mgOnGpu::cppAlign } ); // TEST MISALIGNMENT! } }; @@ -58,8 +59,10 @@ template inline std::unique_ptr> hstMakeUnique(std::size_t N) { // See https://www.bfilipek.com/2019/08/newnew-align.html#requesting-different-alignment return std::unique_ptr>{ new( std::align_val_t{ mgOnGpu::cppAlign } ) T[N]() }; + //return std::unique_ptr>{ new( std::align_val_t{ mgOnGpu::cppAlign } ) T[N+1]() + 1 }; // TEST MISALIGNMENT! }; +/* #ifdef MGONGPU_CPPSIMD template<> inline @@ -68,6 +71,7 @@ std::unique_ptr> hstMakeUnique(std::size_t N }; #endif +*/ #endif diff --git a/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc b/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc index d49bf181ce..ece5097c11 100644 --- a/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc +++ b/epochX/cudacpp/ee_mumu.auto/SubProcesses/P1_Sigma_sm_epem_mupmum/CPPProcess.cc @@ -406,7 +406,7 @@ namespace Proc , const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) { assert( (size_t)(allmomenta) % mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment - //assert( (size_t)(allMEs) % mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment + assert( (size_t)(allMEs) % mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment const int maxtry0 = ( neppV > 16 ? neppV : 16 ); // 16, but at least neppV (otherwise the npagV loop does not even start) fptype allMEsLast[maxtry0] = { 0 }; const int maxtry = std::min( maxtry0, nevt ); // 16, but at most nevt (avoid invalid memory access if nevt Date: Wed, 1 Dec 2021 22:08:52 +0100 Subject: [PATCH 30/34] [klas5X] copy logs ./tput/copyLogs.sh -m2a -eemumu --- .../log_eemumu_auto_d_inl0.txt | 106 +++++++++++------- .../log_eemumu_auto_d_inl1.txt | 106 +++++++++++------- .../log_eemumu_auto_f_inl0.txt | 106 +++++++++++------- .../log_eemumu_auto_f_inl1.txt | 106 +++++++++++------- 4 files changed, 256 insertions(+), 168 deletions(-) diff --git a/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_d_inl0.txt b/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_d_inl0.txt index 24715a4b4e..71d8f86e70 100644 --- a/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_d_inl0.txt +++ b/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_d_inl0.txt @@ -1,59 +1,81 @@ /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum OMPFLAGS= +AVX=512y +FPTYPE=d +HELINL=0 +RNDGEN=curdev +Building in BUILDDIR=build.512y_d_inl0 for tag=512y_d_inl0_curdev (USEBUILDDIR is set = 1) + +make USEBUILDDIR=1 AVX=none +OMPFLAGS= AVX=none FPTYPE=d HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.none_d_inl0 for tag=none_d_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.none_d_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.none_d_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=sse4 OMPFLAGS= AVX=sse4 FPTYPE=d HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.sse4_d_inl0 for tag=sse4_d_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.sse4_d_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.sse4_d_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=avx2 OMPFLAGS= AVX=avx2 FPTYPE=d HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.avx2_d_inl0 for tag=avx2_d_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.avx2_d_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.avx2_d_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=512y OMPFLAGS= AVX=512y FPTYPE=d HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.512y_d_inl0 for tag=512y_d_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512y_d_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.512y_d_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=512z OMPFLAGS= AVX=512z FPTYPE=d HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.512z_d_inl0 for tag=512z_d_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512z_d_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.512z_d_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' -DATE: 2021-12-01_10:07:10 +DATE: 2021-12-01_21:58:04 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl0/gcheck.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 6.534485e+08 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.116555e+09 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 5.957319e+08 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.110528e+09 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 0.709466 sec - 371,909,202 cycles:u # 0.399 GHz - 697,385,836 instructions:u # 1.88 insn per cycle - 0.996526017 seconds time elapsed +TOTAL : 0.732748 sec + 396,994,334 cycles:u # 0.412 GHz + 739,059,272 instructions:u # 1.86 insn per cycle + 1.028274926 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 130 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... @@ -62,13 +84,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.337315e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.337315e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.354112e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.354112e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 7.047599 sec - 18,666,706,719 cycles:u # 2.647 GHz - 48,224,158,012 instructions:u # 2.58 insn per cycle - 7.055088398 seconds time elapsed +TOTAL : 6.991839 sec + 18,522,301,845 cycles:u # 2.648 GHz + 48,224,155,362 instructions:u # 2.60 insn per cycle + 7.000208392 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 621) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl0/runTest.exe @@ -78,13 +100,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[2] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 2.565767e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 2.565767e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 2.564081e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 2.564081e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 4.823406 sec - 12,729,876,419 cycles:u # 2.637 GHz - 29,850,563,434 instructions:u # 2.34 insn per cycle - 4.831352437 seconds time elapsed +TOTAL : 4.836123 sec + 12,759,577,192 cycles:u # 2.636 GHz + 29,850,564,087 instructions:u # 2.34 insn per cycle + 4.844162095 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 3297) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_d_inl0/runTest.exe @@ -94,13 +116,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 4.556887e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.556887e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.565133e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.565133e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 3.663924 sec - 9,087,880,283 cycles:u # 2.478 GHz - 16,644,667,763 instructions:u # 1.83 insn per cycle - 3.671637372 seconds time elapsed +TOTAL : 3.673703 sec + 9,115,434,244 cycles:u # 2.478 GHz + 16,644,668,367 instructions:u # 1.83 insn per cycle + 3.681693279 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2831) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_d_inl0/runTest.exe @@ -110,13 +132,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 4.921532e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.921532e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.909323e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.909323e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 3.602738 sec - 8,968,063,737 cycles:u # 2.486 GHz - 16,531,423,183 instructions:u # 1.84 insn per cycle - 3.610374997 seconds time elapsed +TOTAL : 3.594048 sec + 8,942,444,328 cycles:u # 2.485 GHz + 16,531,423,242 instructions:u # 1.85 insn per cycle + 3.602091761 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2728) (512y: 52) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_d_inl0/runTest.exe @@ -126,13 +148,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 3.828049e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.828049e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 3.676735e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.676735e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 3.958997 sec - 8,638,325,570 cycles:u # 2.180 GHz - 13,280,272,453 instructions:u # 1.54 insn per cycle - 3.966662948 seconds time elapsed +TOTAL : 4.049430 sec + 8,791,223,746 cycles:u # 2.169 GHz + 13,280,272,515 instructions:u # 1.51 insn per cycle + 4.057343967 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1319) (512y: 66) (512z: 2147) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_d_inl0/runTest.exe diff --git a/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_d_inl1.txt b/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_d_inl1.txt index 17c1e59cc3..e453d683c2 100644 --- a/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_d_inl1.txt +++ b/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_d_inl1.txt @@ -1,59 +1,81 @@ /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum OMPFLAGS= +AVX=512y +FPTYPE=d +HELINL=1 +RNDGEN=curdev +Building in BUILDDIR=build.512y_d_inl1 for tag=512y_d_inl1_curdev (USEBUILDDIR is set = 1) + +make USEBUILDDIR=1 AVX=none +OMPFLAGS= AVX=none FPTYPE=d HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.none_d_inl1 for tag=none_d_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.none_d_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.none_d_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=sse4 OMPFLAGS= AVX=sse4 FPTYPE=d HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.sse4_d_inl1 for tag=sse4_d_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.sse4_d_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.sse4_d_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=avx2 OMPFLAGS= AVX=avx2 FPTYPE=d HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.avx2_d_inl1 for tag=avx2_d_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.avx2_d_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.avx2_d_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=512y OMPFLAGS= AVX=512y FPTYPE=d HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.512y_d_inl1 for tag=512y_d_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512y_d_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.512y_d_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=512z OMPFLAGS= AVX=512z FPTYPE=d HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.512z_d_inl1 for tag=512z_d_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512z_d_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.512z_d_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' -DATE: 2021-12-01_10:07:46 +DATE: 2021-12-01_21:58:39 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl1/gcheck.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 6.494723e+08 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.109138e+09 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 5.919373e+08 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.109577e+09 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 1.226259 sec - 997,952,337 cycles:u # 0.839 GHz - 1,981,968,331 instructions:u # 1.99 insn per cycle - 1.518449844 seconds time elapsed +TOTAL : 0.729794 sec + 394,330,255 cycles:u # 0.410 GHz + 734,519,727 instructions:u # 1.86 insn per cycle + 1.026025161 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 130 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... @@ -62,13 +84,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 4.736845e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.736845e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.726360e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.726360e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 3.677158 sec - 9,670,909,975 cycles:u # 2.627 GHz - 17,889,570,187 instructions:u # 1.85 insn per cycle - 3.684775451 seconds time elapsed +TOTAL : 3.668891 sec + 9,654,425,372 cycles:u # 2.628 GHz + 17,889,569,619 instructions:u # 1.85 insn per cycle + 3.677068240 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 2595) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_d_inl1/runTest.exe @@ -78,13 +100,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[2] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 9.455274e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 9.455274e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 9.381860e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 9.381860e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 3.046095 sec - 7,995,654,626 cycles:u # 2.622 GHz - 13,964,472,664 instructions:u # 1.75 insn per cycle - 3.053929213 seconds time elapsed +TOTAL : 3.052269 sec + 8,011,338,634 cycles:u # 2.620 GHz + 13,964,473,137 instructions:u # 1.74 insn per cycle + 3.060276458 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 432) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_d_inl1/runTest.exe @@ -94,13 +116,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.303904e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.303904e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.293550e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.293550e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 2.773556 sec - 7,073,285,248 cycles:u # 2.547 GHz - 11,086,106,492 instructions:u # 1.57 insn per cycle - 2.781104378 seconds time elapsed +TOTAL : 2.784430 sec + 7,096,839,537 cycles:u # 2.544 GHz + 11,086,106,832 instructions:u # 1.56 insn per cycle + 2.792751894 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 461) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_d_inl1/runTest.exe @@ -110,13 +132,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.383099e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.383099e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.359076e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.359076e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 2.767352 sec - 7,074,744,001 cycles:u # 2.552 GHz - 10,919,382,397 instructions:u # 1.54 insn per cycle - 2.775205710 seconds time elapsed +TOTAL : 2.776058 sec + 7,083,525,857 cycles:u # 2.548 GHz + 10,919,382,570 instructions:u # 1.54 insn per cycle + 2.784045302 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 412) (512y: 1) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_d_inl1/runTest.exe @@ -126,13 +148,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 9.344111e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 9.344111e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 9.087523e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 9.087523e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270315e-06 ) GeV^0 -TOTAL : 3.015544 sec - 7,170,726,069 cycles:u # 2.377 GHz - 10,478,969,947 instructions:u # 1.46 insn per cycle - 3.023095729 seconds time elapsed +TOTAL : 3.037580 sec + 7,216,761,782 cycles:u # 2.372 GHz + 10,478,969,816 instructions:u # 1.45 insn per cycle + 3.045604626 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 193) (512y: 2) (512z: 230) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_d_inl1/runTest.exe diff --git a/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_f_inl0.txt b/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_f_inl0.txt index 30e3d856fb..e61869d016 100644 --- a/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_f_inl0.txt +++ b/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_f_inl0.txt @@ -1,59 +1,81 @@ /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum OMPFLAGS= +AVX=512y +FPTYPE=f +HELINL=0 +RNDGEN=curdev +Building in BUILDDIR=build.512y_f_inl0 for tag=512y_f_inl0_curdev (USEBUILDDIR is set = 1) + +make USEBUILDDIR=1 AVX=none +OMPFLAGS= AVX=none FPTYPE=f HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.none_f_inl0 for tag=none_f_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.none_f_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.none_f_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=sse4 OMPFLAGS= AVX=sse4 FPTYPE=f HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.sse4_f_inl0 for tag=sse4_f_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.sse4_f_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.sse4_f_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=avx2 OMPFLAGS= AVX=avx2 FPTYPE=f HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.avx2_f_inl0 for tag=avx2_f_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.avx2_f_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.avx2_f_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=512y OMPFLAGS= AVX=512y FPTYPE=f HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.512y_f_inl0 for tag=512y_f_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512y_f_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.512y_f_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=512z OMPFLAGS= AVX=512z FPTYPE=f HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.512z_f_inl0 for tag=512z_f_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512z_f_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.512z_f_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' -DATE: 2021-12-01_10:08:12 +DATE: 2021-12-01_21:59:06 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl0/gcheck.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=2, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 1.567154e+09 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.236935e+09 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.427625e+09 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.256233e+09 ) sec^-1 MeanMatrixElemValue = ( 1.371686e-02 +- 3.270219e-06 ) GeV^0 -TOTAL : 0.842427 sec - 716,989,672 cycles:u # 0.704 GHz - 1,484,422,831 instructions:u # 2.07 insn per cycle - 1.126361376 seconds time elapsed +TOTAL : 0.669291 sec + 340,663,519 cycles:u # 0.384 GHz + 672,342,639 instructions:u # 1.97 insn per cycle + 0.967024151 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 56 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... @@ -62,13 +84,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=6, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.224589e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.224589e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.227750e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.227750e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371707e-02 +- 3.270376e-06 ) GeV^0 -TOTAL : 7.053663 sec - 18,764,093,689 cycles:u # 2.659 GHz - 47,311,642,026 instructions:u # 2.52 insn per cycle - 7.060851302 seconds time elapsed +TOTAL : 7.045155 sec + 18,729,859,154 cycles:u # 2.658 GHz + 47,311,640,742 instructions:u # 2.53 insn per cycle + 7.052949347 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 559) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl0/runTest.exe @@ -78,13 +100,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=6, zero=0) Internal loops fptype_sv = VECTOR[4] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 4.573579e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.573579e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.583214e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.583214e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270375e-06 ) GeV^0 -TOTAL : 3.344023 sec - 8,846,286,898 cycles:u # 2.642 GHz - 19,615,196,579 instructions:u # 2.22 insn per cycle - 3.351351467 seconds time elapsed +TOTAL : 3.370620 sec + 8,915,711,214 cycles:u # 2.642 GHz + 19,615,194,859 instructions:u # 2.20 insn per cycle + 3.378548883 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 3969) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_f_inl0/runTest.exe @@ -94,13 +116,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=5, zero=0) Internal loops fptype_sv = VECTOR[8] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 8.181232e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 8.181232e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 8.183244e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 8.183244e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371705e-02 +- 3.270339e-06 ) GeV^0 -TOTAL : 2.698787 sec - 6,823,046,265 cycles:u # 2.524 GHz - 12,524,650,277 instructions:u # 1.84 insn per cycle - 2.705979783 seconds time elapsed +TOTAL : 2.684441 sec + 6,783,050,356 cycles:u # 2.522 GHz + 12,524,650,491 instructions:u # 1.85 insn per cycle + 2.692018765 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 3175) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_f_inl0/runTest.exe @@ -110,13 +132,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=5, zero=0) Internal loops fptype_sv = VECTOR[8] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 8.961058e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 8.961058e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 8.980040e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 8.980040e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371705e-02 +- 3.270339e-06 ) GeV^0 -TOTAL : 2.587116 sec - 6,548,070,094 cycles:u # 2.527 GHz - 12,477,465,603 instructions:u # 1.91 insn per cycle - 2.594729483 seconds time elapsed +TOTAL : 2.606645 sec + 6,599,980,765 cycles:u # 2.527 GHz + 12,477,467,361 instructions:u # 1.89 insn per cycle + 2.614446054 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 3069) (512y: 26) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_f_inl0/runTest.exe @@ -126,13 +148,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=5, zero=0) Internal loops fptype_sv = VECTOR[16] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 7.493309e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 7.493309e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 7.346633e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 7.346633e+06 ) sec^-1 MeanMatrixElemValue = ( 1.371705e-02 +- 3.270340e-06 ) GeV^0 -TOTAL : 2.727509 sec - 6,281,035,658 cycles:u # 2.300 GHz - 10,798,421,531 instructions:u # 1.72 insn per cycle - 2.734747652 seconds time elapsed +TOTAL : 2.745844 sec + 6,303,361,200 cycles:u # 2.291 GHz + 10,798,421,576 instructions:u # 1.71 insn per cycle + 2.753608906 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1772) (512y: 12) (512z: 2256) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_f_inl0/runTest.exe diff --git a/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_f_inl1.txt b/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_f_inl1.txt index adffd368b3..e405135d2a 100644 --- a/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_f_inl1.txt +++ b/epochX/cudacpp/tput/logs_eemumu_auto/log_eemumu_auto_f_inl1.txt @@ -1,59 +1,81 @@ /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum OMPFLAGS= +AVX=512y +FPTYPE=f +HELINL=1 +RNDGEN=curdev +Building in BUILDDIR=build.512y_f_inl1 for tag=512y_f_inl1_curdev (USEBUILDDIR is set = 1) + +make USEBUILDDIR=1 AVX=none +OMPFLAGS= AVX=none FPTYPE=f HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.none_f_inl1 for tag=none_f_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.none_f_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.none_f_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=sse4 OMPFLAGS= AVX=sse4 FPTYPE=f HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.sse4_f_inl1 for tag=sse4_f_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.sse4_f_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.sse4_f_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=avx2 OMPFLAGS= AVX=avx2 FPTYPE=f HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.avx2_f_inl1 for tag=avx2_f_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.avx2_f_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.avx2_f_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=512y OMPFLAGS= AVX=512y FPTYPE=f HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.512y_f_inl1 for tag=512y_f_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512y_f_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.512y_f_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make USEBUILDDIR=1 AVX=512z OMPFLAGS= AVX=512z FPTYPE=f HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.512z_f_inl1 for tag=512z_f_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512z_f_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' +make[1]: Nothing to be done for `all.512z_f_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum' -DATE: 2021-12-01_10:08:43 +DATE: 2021-12-01_21:59:36 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl1/gcheck.exe -p 2048 256 12 OMP= Process = SIGMA_SM_EPEM_MUPMUM_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=2, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 1.577348e+09 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.256680e+09 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.415544e+09 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.241284e+09 ) sec^-1 MeanMatrixElemValue = ( 1.371686e-02 +- 3.270219e-06 ) GeV^0 -TOTAL : 0.907275 sec - 718,084,580 cycles:u # 0.706 GHz - 1,482,928,433 instructions:u # 2.07 insn per cycle - 1.190328356 seconds time elapsed +TOTAL : 0.653274 sec + 338,494,738 cycles:u # 0.386 GHz + 664,838,163 instructions:u # 1.96 insn per cycle + 0.939786552 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 56 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... @@ -62,13 +84,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=6, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.203281e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.203281e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.200711e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.200711e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270375e-06 ) GeV^0 -TOTAL : 2.431255 sec - 6,417,013,841 cycles:u # 2.634 GHz - 12,595,066,654 instructions:u # 1.96 insn per cycle - 2.438826820 seconds time elapsed +TOTAL : 2.443712 sec + 6,450,825,445 cycles:u # 2.634 GHz + 12,595,066,498 instructions:u # 1.95 insn per cycle + 2.451707004 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 796) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.none_f_inl1/runTest.exe @@ -78,13 +100,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=6, zero=0) Internal loops fptype_sv = VECTOR[4] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.923450e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.923450e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.905316e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.905316e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371706e-02 +- 3.270375e-06 ) GeV^0 -TOTAL : 2.297739 sec - 6,058,657,245 cycles:u # 2.632 GHz - 11,389,032,042 instructions:u # 1.88 insn per cycle - 2.304986309 seconds time elapsed +TOTAL : 2.306185 sec + 6,073,875,427 cycles:u # 2.628 GHz + 11,389,030,202 instructions:u # 1.88 insn per cycle + 2.313780178 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 508) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.sse4_f_inl1/runTest.exe @@ -94,13 +116,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=5, zero=0) Internal loops fptype_sv = VECTOR[8] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 2.894889e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 2.894889e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 2.883390e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 2.883390e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371705e-02 +- 3.270339e-06 ) GeV^0 -TOTAL : 2.105405 sec - 5,454,622,456 cycles:u # 2.585 GHz - 9,655,713,280 instructions:u # 1.77 insn per cycle - 2.113229620 seconds time elapsed +TOTAL : 2.128015 sec + 5,517,380,840 cycles:u # 2.587 GHz + 9,655,713,434 instructions:u # 1.75 insn per cycle + 2.136040937 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 533) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.avx2_f_inl1/runTest.exe @@ -110,13 +132,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=5, zero=0) Internal loops fptype_sv = VECTOR[8] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 3.028386e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.028386e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 2.938130e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 2.938130e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371705e-02 +- 3.270339e-06 ) GeV^0 -TOTAL : 2.093570 sec - 5,428,962,159 cycles:u # 2.588 GHz - 9,583,362,532 instructions:u # 1.77 insn per cycle - 2.101081895 seconds time elapsed +TOTAL : 2.123205 sec + 5,504,030,300 cycles:u # 2.587 GHz + 9,583,364,167 instructions:u # 1.74 insn per cycle + 2.130848906 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 497) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512y_f_inl1/runTest.exe @@ -126,13 +148,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/ Process = SIGMA_SM_EPEM_MUPMUM_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=5, zero=0) Internal loops fptype_sv = VECTOR[16] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 2.059238e+07 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 2.059238e+07 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.982221e+07 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.982221e+07 ) sec^-1 MeanMatrixElemValue = ( 1.371705e-02 +- 3.270340e-06 ) GeV^0 -TOTAL : 2.192812 sec - 5,434,991,260 cycles:u # 2.474 GHz - 9,332,494,253 instructions:u # 1.72 insn per cycle - 2.200097462 seconds time elapsed +TOTAL : 2.204938 sec + 5,457,154,323 cycles:u # 2.470 GHz + 9,332,494,167 instructions:u # 1.71 insn per cycle + 2.212867763 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 293) (512y: 0) (512z: 251) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/ee_mumu/SubProcesses/P1_Sigma_sm_epem_mupmum/build.512z_f_inl1/runTest.exe From 1e4e64e2496f3c9e355367df1362f2fb42c499fd Mon Sep 17 00:00:00 2001 From: Andrea Valassi Date: Wed, 1 Dec 2021 22:09:57 +0100 Subject: [PATCH 31/34] [klas5X] regenerate ggtt and ggttgg auto, resync to ggttgg manual --- .../gg_tt.auto/CODEGEN_cudacpp_gg_tt_log.txt | 10 +++++----- epochX/cudacpp/gg_tt.auto/SubProcesses/Memory.h | 4 ++++ .../SubProcesses/P1_Sigma_sm_gg_ttx/CPPProcess.cc | 5 ++++- epochX/cudacpp/gg_tt.auto/src/mgOnGpuConfig.h | 7 ++++--- .../gg_ttgg.auto/CODEGEN_cudacpp_gg_ttgg_log.txt | 13 +++++++------ epochX/cudacpp/gg_ttgg.auto/SubProcesses/Memory.h | 4 ++++ .../SubProcesses/P1_Sigma_sm_gg_ttxgg/CPPProcess.cc | 5 ++++- epochX/cudacpp/gg_ttgg.auto/src/mgOnGpuConfig.h | 7 ++++--- epochX/cudacpp/gg_ttgg/SubProcesses/Memory.h | 4 ++++ .../SubProcesses/P1_Sigma_sm_gg_ttxgg/CPPProcess.cc | 5 ++++- epochX/cudacpp/gg_ttgg/src/mgOnGpuConfig.h | 7 ++++--- 11 files changed, 48 insertions(+), 23 deletions(-) diff --git a/epochX/cudacpp/gg_tt.auto/CODEGEN_cudacpp_gg_tt_log.txt b/epochX/cudacpp/gg_tt.auto/CODEGEN_cudacpp_gg_tt_log.txt index 23aca5f4a5..bd5e8b971a 100644 --- a/epochX/cudacpp/gg_tt.auto/CODEGEN_cudacpp_gg_tt_log.txt +++ b/epochX/cudacpp/gg_tt.auto/CODEGEN_cudacpp_gg_tt_log.txt @@ -50,7 +50,7 @@ generate g g > t t~ No model currently active, so we import the Standard Model INFO: load particles INFO: load vertices -DEBUG: model prefixing takes 0.006844043731689453  +DEBUG: model prefixing takes 0.006895780563354492  INFO: Restrict model sm with file models/sm/restrict_default.dat . INFO: Change particles name to pass to MG5 convention DEBUG: True [misc.py at line 2192]  @@ -67,7 +67,7 @@ INFO: Please specify coupling orders to bypass this step. INFO: Trying coupling order WEIGHTED<=2: WEIGTHED IS QCD+2*QED INFO: Trying process: g g > t t~ WEIGHTED<=2 @1 INFO: Process has 3 diagrams -1 processes with 3 diagrams generated in 0.010 s +1 processes with 3 diagrams generated in 0.094 s Total: 1 processes with 3 diagrams output standalone_cudacpp CODEGEN_cudacpp_gg_tt Output will be done with PLUGIN: CUDACPP_SA_OUTPUT @@ -118,6 +118,6 @@ INFO: /data/avalassi/GPU2020/MG5aMC/2.7.0_gpu/CODEGEN_cudacpp_gg_tt/src/. and /d DEBUG: Entering PLUGIN_ProcessExporter.finalize [output.py at line 154]  quit -real 0m3.958s -user 0m0.792s -sys 0m0.129s +real 0m4.237s +user 0m0.789s +sys 0m0.142s diff --git a/epochX/cudacpp/gg_tt.auto/SubProcesses/Memory.h b/epochX/cudacpp/gg_tt.auto/SubProcesses/Memory.h index a9c08d3dd0..34b5e02071 100644 --- a/epochX/cudacpp/gg_tt.auto/SubProcesses/Memory.h +++ b/epochX/cudacpp/gg_tt.auto/SubProcesses/Memory.h @@ -51,6 +51,7 @@ template struct CppHstDeleter { void operator()(T* mem) { ::operator delete( mem, std::align_val_t{ mgOnGpu::cppAlign } ); + //::operator delete( mem-1, std::align_val_t{ mgOnGpu::cppAlign } ); // TEST MISALIGNMENT! } }; @@ -58,8 +59,10 @@ template inline std::unique_ptr> hstMakeUnique(std::size_t N) { // See https://www.bfilipek.com/2019/08/newnew-align.html#requesting-different-alignment return std::unique_ptr>{ new( std::align_val_t{ mgOnGpu::cppAlign } ) T[N]() }; + //return std::unique_ptr>{ new( std::align_val_t{ mgOnGpu::cppAlign } ) T[N+1]() + 1 }; // TEST MISALIGNMENT! }; +/* #ifdef MGONGPU_CPPSIMD template<> inline @@ -68,6 +71,7 @@ std::unique_ptr> hstMakeUnique(std::size_t N }; #endif +*/ #endif diff --git a/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/CPPProcess.cc b/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/CPPProcess.cc index 2d37aff5ea..77631c4135 100644 --- a/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/CPPProcess.cc +++ b/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/CPPProcess.cc @@ -411,7 +411,7 @@ namespace Proc , const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) { assert( (size_t)(allmomenta) % mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment - //assert( (size_t)(allMEs) % mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment + assert( (size_t)(allMEs) % mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment const int maxtry0 = ( neppV > 16 ? neppV : 16 ); // 16, but at least neppV (otherwise the npagV loop does not even start) fptype allMEsLast[maxtry0] = { 0 }; const int maxtry = std::min( maxtry0, nevt ); // 16, but at most nevt (avoid invalid memory access if nevt t t~ g g No model currently active, so we import the Standard Model INFO: load particles INFO: load vertices -DEBUG: model prefixing takes 0.006913185119628906  +DEBUG: model prefixing takes 0.029345273971557617  INFO: Restrict model sm with file models/sm/restrict_default.dat . INFO: Change particles name to pass to MG5 convention DEBUG: True [misc.py at line 2192]  @@ -67,7 +68,7 @@ INFO: Please specify coupling orders to bypass this step. INFO: Trying coupling order WEIGHTED<=4: WEIGTHED IS QCD+2*QED INFO: Trying process: g g > t t~ g g WEIGHTED<=4 @1 INFO: Process has 123 diagrams -1 processes with 123 diagrams generated in 0.209 s +1 processes with 123 diagrams generated in 0.630 s Total: 1 processes with 123 diagrams output standalone_cudacpp CODEGEN_cudacpp_gg_ttgg Output will be done with PLUGIN: CUDACPP_SA_OUTPUT @@ -100,7 +101,7 @@ INFO: Created files CPPProcess.h and CPPProcess.cc in directory /data/avalassi/G DEBUG: Entering PLUGIN_OneProcessExporter.edit_check_sa [model_handling.py at line 907]  DEBUG: Entering PLUGIN_OneProcessExporter.edit_mgonGPU [model_handling.py at line 922]  DEBUG: Entering PLUGIN_OneProcessExporter.edit_processidfile [model_handling.py at line 929]  -Generated helas calls for 1 subprocesses (123 diagrams) in 0.557 s +Generated helas calls for 1 subprocesses (123 diagrams) in 1.734 s DEBUG: Entering PLUGIN_ProcessExporter.convert_model (create the model) [output.py at line 145]  ALOHA: aloha creates routines (starting by VVV1) VVV1 @@ -138,6 +139,6 @@ INFO: /data/avalassi/GPU2020/MG5aMC/2.7.0_gpu/CODEGEN_cudacpp_gg_ttgg/src/. and DEBUG: Entering PLUGIN_ProcessExporter.finalize [output.py at line 154]  quit -real 0m5.154s -user 0m1.994s -sys 0m0.138s +real 0m13.206s +user 0m2.108s +sys 0m0.156s diff --git a/epochX/cudacpp/gg_ttgg.auto/SubProcesses/Memory.h b/epochX/cudacpp/gg_ttgg.auto/SubProcesses/Memory.h index a9c08d3dd0..34b5e02071 100644 --- a/epochX/cudacpp/gg_ttgg.auto/SubProcesses/Memory.h +++ b/epochX/cudacpp/gg_ttgg.auto/SubProcesses/Memory.h @@ -51,6 +51,7 @@ template struct CppHstDeleter { void operator()(T* mem) { ::operator delete( mem, std::align_val_t{ mgOnGpu::cppAlign } ); + //::operator delete( mem-1, std::align_val_t{ mgOnGpu::cppAlign } ); // TEST MISALIGNMENT! } }; @@ -58,8 +59,10 @@ template inline std::unique_ptr> hstMakeUnique(std::size_t N) { // See https://www.bfilipek.com/2019/08/newnew-align.html#requesting-different-alignment return std::unique_ptr>{ new( std::align_val_t{ mgOnGpu::cppAlign } ) T[N]() }; + //return std::unique_ptr>{ new( std::align_val_t{ mgOnGpu::cppAlign } ) T[N+1]() + 1 }; // TEST MISALIGNMENT! }; +/* #ifdef MGONGPU_CPPSIMD template<> inline @@ -68,6 +71,7 @@ std::unique_ptr> hstMakeUnique(std::size_t N }; #endif +*/ #endif diff --git a/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/CPPProcess.cc b/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/CPPProcess.cc index c4bd7eeb18..d5867a59ae 100644 --- a/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/CPPProcess.cc +++ b/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/CPPProcess.cc @@ -2176,7 +2176,7 @@ namespace Proc , const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) { assert( (size_t)(allmomenta) % mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment - //assert( (size_t)(allMEs) % mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment + assert( (size_t)(allMEs) % mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment const int maxtry0 = ( neppV > 16 ? neppV : 16 ); // 16, but at least neppV (otherwise the npagV loop does not even start) fptype allMEsLast[maxtry0] = { 0 }; const int maxtry = std::min( maxtry0, nevt ); // 16, but at most nevt (avoid invalid memory access if nevt struct CppHstDeleter { void operator()(T* mem) { ::operator delete( mem, std::align_val_t{ mgOnGpu::cppAlign } ); + //::operator delete( mem-1, std::align_val_t{ mgOnGpu::cppAlign } ); // TEST MISALIGNMENT! } }; @@ -58,8 +59,10 @@ template inline std::unique_ptr> hstMakeUnique(std::size_t N) { // See https://www.bfilipek.com/2019/08/newnew-align.html#requesting-different-alignment return std::unique_ptr>{ new( std::align_val_t{ mgOnGpu::cppAlign } ) T[N]() }; + //return std::unique_ptr>{ new( std::align_val_t{ mgOnGpu::cppAlign } ) T[N+1]() + 1 }; // TEST MISALIGNMENT! }; +/* #ifdef MGONGPU_CPPSIMD template<> inline @@ -68,6 +71,7 @@ std::unique_ptr> hstMakeUnique(std::size_t N }; #endif +*/ #endif diff --git a/epochX/cudacpp/gg_ttgg/SubProcesses/P1_Sigma_sm_gg_ttxgg/CPPProcess.cc b/epochX/cudacpp/gg_ttgg/SubProcesses/P1_Sigma_sm_gg_ttxgg/CPPProcess.cc index c4bd7eeb18..d5867a59ae 100644 --- a/epochX/cudacpp/gg_ttgg/SubProcesses/P1_Sigma_sm_gg_ttxgg/CPPProcess.cc +++ b/epochX/cudacpp/gg_ttgg/SubProcesses/P1_Sigma_sm_gg_ttxgg/CPPProcess.cc @@ -2176,7 +2176,7 @@ namespace Proc , const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) { assert( (size_t)(allmomenta) % mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment - //assert( (size_t)(allMEs) % mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment + assert( (size_t)(allMEs) % mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment const int maxtry0 = ( neppV > 16 ? neppV : 16 ); // 16, but at least neppV (otherwise the npagV loop does not even start) fptype allMEsLast[maxtry0] = { 0 }; const int maxtry = std::min( maxtry0, nevt ); // 16, but at most nevt (avoid invalid memory access if nevt Date: Wed, 1 Dec 2021 22:15:25 +0100 Subject: [PATCH 32/34] [klas5X] rerun ggtt auto tests - performance seems exactly the same within fluctuations up and down --- .../logs_ggtt_auto/log_ggtt_auto_d_inl0.txt | 106 +++++++++++------- .../logs_ggtt_auto/log_ggtt_auto_d_inl1.txt | 106 +++++++++++------- .../logs_ggtt_auto/log_ggtt_auto_f_inl0.txt | 106 +++++++++++------- .../logs_ggtt_auto/log_ggtt_auto_f_inl1.txt | 106 +++++++++++------- 4 files changed, 256 insertions(+), 168 deletions(-) diff --git a/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_d_inl0.txt b/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_d_inl0.txt index f223ac29b8..655db629f2 100644 --- a/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_d_inl0.txt +++ b/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_d_inl0.txt @@ -1,59 +1,81 @@ /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx OMPFLAGS= +AVX=512y +FPTYPE=d +HELINL=0 +RNDGEN=curdev +Building in BUILDDIR=build.512y_d_inl0 for tag=512y_d_inl0_curdev (USEBUILDDIR is set = 1) + +make USEBUILDDIR=1 AVX=none +OMPFLAGS= AVX=none FPTYPE=d HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.none_d_inl0 for tag=none_d_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.none_d_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make[1]: Nothing to be done for `all.none_d_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make USEBUILDDIR=1 AVX=sse4 OMPFLAGS= AVX=sse4 FPTYPE=d HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.sse4_d_inl0 for tag=sse4_d_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.sse4_d_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make[1]: Nothing to be done for `all.sse4_d_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make USEBUILDDIR=1 AVX=avx2 OMPFLAGS= AVX=avx2 FPTYPE=d HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.avx2_d_inl0 for tag=avx2_d_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.avx2_d_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make[1]: Nothing to be done for `all.avx2_d_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make USEBUILDDIR=1 AVX=512y OMPFLAGS= AVX=512y FPTYPE=d HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.512y_d_inl0 for tag=512y_d_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512y_d_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make[1]: Nothing to be done for `all.512y_d_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make USEBUILDDIR=1 AVX=512z OMPFLAGS= AVX=512z FPTYPE=d HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.512z_d_inl0 for tag=512z_d_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512z_d_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make[1]: Nothing to be done for `all.512z_d_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' -DATE: 2021-12-01_10:48:37 +DATE: 2021-12-01_22:13:25 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.none_d_inl0/gcheck.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTX_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 1.312795e+08 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.432637e+08 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.285880e+08 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.432348e+08 ) sec^-1 MeanMatrixElemValue = ( 2.085623e+00 +- 4.835084e-03 ) GeV^0 -TOTAL : 1.113676 sec - 1,224,992,382 cycles:u # 1.002 GHz - 2,374,241,054 instructions:u # 1.94 insn per cycle - 1.402465765 seconds time elapsed +TOTAL : 0.564558 sec + 134,313,582 cycles:u # 0.167 GHz + 170,263,141 instructions:u # 1.27 insn per cycle + 0.869732961 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 172 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... @@ -62,13 +84,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 2.010504e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 2.010504e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 2.009352e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 2.009352e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085623e+00 +- 4.835084e-03 ) GeV^0 -TOTAL : 2.835012 sec - 7,496,138,252 cycles:u # 2.641 GHz - 22,151,432,219 instructions:u # 2.96 insn per cycle - 2.842413425 seconds time elapsed +TOTAL : 2.839267 sec + 7,502,043,285 cycles:u # 2.639 GHz + 22,151,432,631 instructions:u # 2.95 insn per cycle + 2.846831212 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 552) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.none_d_inl0/runTest.exe @@ -78,13 +100,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[2] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 3.152961e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.152961e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 3.156017e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.156017e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085623e+00 +- 4.835084e-03 ) GeV^0 -TOTAL : 1.892329 sec - 4,974,774,276 cycles:u # 2.622 GHz - 13,213,234,724 instructions:u # 2.66 insn per cycle - 1.900209238 seconds time elapsed +TOTAL : 1.892782 sec + 4,976,565,571 cycles:u # 2.623 GHz + 13,213,234,924 instructions:u # 2.66 insn per cycle + 1.900524912 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 2602) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.sse4_d_inl0/runTest.exe @@ -94,13 +116,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 5.578043e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 5.578043e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 5.597704e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 5.597704e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085623e+00 +- 4.835084e-03 ) GeV^0 -TOTAL : 1.163111 sec - 2,648,976,099 cycles:u # 2.269 GHz - 5,240,076,824 instructions:u # 1.98 insn per cycle - 1.170428063 seconds time elapsed +TOTAL : 1.164764 sec + 2,643,954,618 cycles:u # 2.262 GHz + 5,240,076,573 instructions:u # 1.98 insn per cycle + 1.172528990 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2299) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.avx2_d_inl0/runTest.exe @@ -110,13 +132,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 6.311632e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 6.311632e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 6.303598e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 6.303598e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085623e+00 +- 4.835084e-03 ) GeV^0 -TOTAL : 1.053746 sec - 2,403,014,525 cycles:u # 2.270 GHz - 5,029,831,866 instructions:u # 2.09 insn per cycle - 1.061475986 seconds time elapsed +TOTAL : 1.058497 sec + 2,410,652,064 cycles:u # 2.267 GHz + 5,029,831,813 instructions:u # 2.09 insn per cycle + 1.066112989 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2170) (512y: 52) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512y_d_inl0/runTest.exe @@ -126,13 +148,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 3.778810e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.778810e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 3.787115e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.787115e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085623e+00 +- 4.835084e-03 ) GeV^0 -TOTAL : 1.613268 sec - 2,705,674,570 cycles:u # 1.673 GHz - 3,394,918,575 instructions:u # 1.25 insn per cycle - 1.620793314 seconds time elapsed +TOTAL : 1.612414 sec + 2,702,600,361 cycles:u # 1.672 GHz + 3,394,919,120 instructions:u # 1.26 insn per cycle + 1.620137335 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1258) (512y: 66) (512z: 1678) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512z_d_inl0/runTest.exe diff --git a/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_d_inl1.txt b/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_d_inl1.txt index 3430f3ea77..7cec347a6f 100644 --- a/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_d_inl1.txt +++ b/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_d_inl1.txt @@ -1,59 +1,81 @@ /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx OMPFLAGS= +AVX=512y +FPTYPE=d +HELINL=1 +RNDGEN=curdev +Building in BUILDDIR=build.512y_d_inl1 for tag=512y_d_inl1_curdev (USEBUILDDIR is set = 1) + +make USEBUILDDIR=1 AVX=none +OMPFLAGS= AVX=none FPTYPE=d HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.none_d_inl1 for tag=none_d_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.none_d_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make[1]: Nothing to be done for `all.none_d_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make USEBUILDDIR=1 AVX=sse4 OMPFLAGS= AVX=sse4 FPTYPE=d HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.sse4_d_inl1 for tag=sse4_d_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.sse4_d_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make[1]: Nothing to be done for `all.sse4_d_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make USEBUILDDIR=1 AVX=avx2 OMPFLAGS= AVX=avx2 FPTYPE=d HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.avx2_d_inl1 for tag=avx2_d_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.avx2_d_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make[1]: Nothing to be done for `all.avx2_d_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make USEBUILDDIR=1 AVX=512y OMPFLAGS= AVX=512y FPTYPE=d HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.512y_d_inl1 for tag=512y_d_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512y_d_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make[1]: Nothing to be done for `all.512y_d_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make USEBUILDDIR=1 AVX=512z OMPFLAGS= AVX=512z FPTYPE=d HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.512z_d_inl1 for tag=512z_d_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512z_d_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make[1]: Nothing to be done for `all.512z_d_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' -DATE: 2021-12-01_10:48:59 +DATE: 2021-12-01_22:13:45 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.none_d_inl1/gcheck.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTX_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 1.306291e+08 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.423126e+08 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.291091e+08 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.432226e+08 ) sec^-1 MeanMatrixElemValue = ( 2.085623e+00 +- 4.835084e-03 ) GeV^0 -TOTAL : 1.024492 sec - 1,222,382,669 cycles:u # 1.010 GHz - 2,372,751,454 instructions:u # 1.94 insn per cycle - 1.312533047 seconds time elapsed +TOTAL : 0.509230 sec + 131,942,461 cycles:u # 0.280 GHz + 169,673,463 instructions:u # 1.29 insn per cycle + 0.800285385 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 172 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... @@ -62,13 +84,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 2.341609e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 2.341609e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 2.358408e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 2.358408e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085623e+00 +- 4.835084e-03 ) GeV^0 -TOTAL : 2.465367 sec - 6,506,955,421 cycles:u # 2.635 GHz - 16,136,621,169 instructions:u # 2.48 insn per cycle - 2.472952243 seconds time elapsed +TOTAL : 2.450886 sec + 6,468,783,207 cycles:u # 2.634 GHz + 16,136,620,215 instructions:u # 2.49 insn per cycle + 2.458334026 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 676) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.none_d_inl1/runTest.exe @@ -78,13 +100,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[2] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 3.112549e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.112549e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 3.118870e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.118870e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085623e+00 +- 4.835084e-03 ) GeV^0 -TOTAL : 1.914624 sec - 5,030,295,164 cycles:u # 2.621 GHz - 10,269,628,611 instructions:u # 2.04 insn per cycle - 1.921809425 seconds time elapsed +TOTAL : 1.918151 sec + 5,029,050,305 cycles:u # 2.621 GHz + 10,269,628,941 instructions:u # 2.04 insn per cycle + 1.925802156 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 2413) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.sse4_d_inl1/runTest.exe @@ -94,13 +116,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 4.678398e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.678398e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.651210e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.651210e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085623e+00 +- 4.835084e-03 ) GeV^0 -TOTAL : 1.341163 sec - 3,052,359,057 cycles:u # 2.271 GHz - 5,123,224,339 instructions:u # 1.68 insn per cycle - 1.348125127 seconds time elapsed +TOTAL : 1.352266 sec + 3,073,586,268 cycles:u # 2.266 GHz + 5,123,225,660 instructions:u # 1.67 insn per cycle + 1.359989085 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2607) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.avx2_d_inl1/runTest.exe @@ -110,13 +132,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 4.759932e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.759932e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.721070e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.721070e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085623e+00 +- 4.835084e-03 ) GeV^0 -TOTAL : 1.324804 sec - 3,018,474,640 cycles:u # 2.271 GHz - 4,738,911,570 instructions:u # 1.57 insn per cycle - 1.332081839 seconds time elapsed +TOTAL : 1.337394 sec + 3,043,536,898 cycles:u # 2.267 GHz + 4,738,911,864 instructions:u # 1.56 insn per cycle + 1.345452870 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2215) (512y: 187) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512y_d_inl1/runTest.exe @@ -126,13 +148,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 3.516536e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.516536e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 3.492611e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.492611e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085623e+00 +- 4.835084e-03 ) GeV^0 -TOTAL : 1.717694 sec - 2,872,588,243 cycles:u # 1.668 GHz - 3,812,896,988 instructions:u # 1.33 insn per cycle - 1.725199428 seconds time elapsed +TOTAL : 1.728387 sec + 2,862,407,429 cycles:u # 1.652 GHz + 3,812,896,954 instructions:u # 1.33 insn per cycle + 1.735977424 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1529) (512y: 256) (512z: 1635) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512z_d_inl1/runTest.exe diff --git a/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_f_inl0.txt b/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_f_inl0.txt index 7182b6503d..715f4fa09a 100644 --- a/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_f_inl0.txt +++ b/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_f_inl0.txt @@ -1,59 +1,81 @@ /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx OMPFLAGS= +AVX=512y +FPTYPE=f +HELINL=0 +RNDGEN=curdev +Building in BUILDDIR=build.512y_f_inl0 for tag=512y_f_inl0_curdev (USEBUILDDIR is set = 1) + +make USEBUILDDIR=1 AVX=none +OMPFLAGS= AVX=none FPTYPE=f HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.none_f_inl0 for tag=none_f_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.none_f_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make[1]: Nothing to be done for `all.none_f_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make USEBUILDDIR=1 AVX=sse4 OMPFLAGS= AVX=sse4 FPTYPE=f HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.sse4_f_inl0 for tag=sse4_f_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.sse4_f_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make[1]: Nothing to be done for `all.sse4_f_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make USEBUILDDIR=1 AVX=avx2 OMPFLAGS= AVX=avx2 FPTYPE=f HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.avx2_f_inl0 for tag=avx2_f_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.avx2_f_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make[1]: Nothing to be done for `all.avx2_f_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make USEBUILDDIR=1 AVX=512y OMPFLAGS= AVX=512y FPTYPE=f HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.512y_f_inl0 for tag=512y_f_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512y_f_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make[1]: Nothing to be done for `all.512y_f_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make USEBUILDDIR=1 AVX=512z OMPFLAGS= AVX=512z FPTYPE=f HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.512z_f_inl0 for tag=512z_f_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512z_f_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make[1]: Nothing to be done for `all.512z_f_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' -DATE: 2021-12-01_10:49:20 +DATE: 2021-12-01_22:14:07 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.none_f_inl0/gcheck.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTX_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 3.383804e+08 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.808356e+08 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 3.328838e+08 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.845146e+08 ) sec^-1 MeanMatrixElemValue = ( 2.085652e+00 +- 4.835323e-03 ) GeV^0 -TOTAL : 0.852993 sec - 741,956,461 cycles:u # 0.736 GHz - 1,483,788,078 instructions:u # 2.00 insn per cycle - 1.143851305 seconds time elapsed +TOTAL : 0.988240 sec + 742,179,391 cycles:u # 0.734 GHz + 1,484,787,193 instructions:u # 2.00 insn per cycle + 1.277373737 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 96 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... @@ -62,13 +84,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.916540e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.916540e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.918354e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.918354e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085710e+00 +- 4.835783e-03 ) GeV^0 -TOTAL : 2.916806 sec - 7,746,567,982 cycles:u # 2.652 GHz - 22,160,206,610 instructions:u # 2.86 insn per cycle - 2.924231787 seconds time elapsed +TOTAL : 2.914965 sec + 7,745,225,991 cycles:u # 2.653 GHz + 22,160,207,394 instructions:u # 2.86 insn per cycle + 2.922552629 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 500) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.none_f_inl0/runTest.exe @@ -78,13 +100,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 4.577346e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.577346e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.574294e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.574294e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085711e+00 +- 4.835793e-03 ) GeV^0 -TOTAL : 1.334202 sec - 3,504,817,250 cycles:u # 2.624 GHz - 8,219,015,975 instructions:u # 2.35 insn per cycle - 1.342614165 seconds time elapsed +TOTAL : 1.330729 sec + 3,507,746,669 cycles:u # 2.626 GHz + 8,219,016,288 instructions:u # 2.34 insn per cycle + 1.338543169 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 3259) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.sse4_f_inl0/runTest.exe @@ -94,13 +116,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.044839e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.044839e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.052437e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.052437e+06 ) sec^-1 MeanMatrixElemValue = ( 2.085720e+00 +- 4.835915e-03 ) GeV^0 -TOTAL : 0.679389 sec - 1,567,021,135 cycles:u # 2.292 GHz - 3,160,253,157 instructions:u # 2.02 insn per cycle - 0.686556102 seconds time elapsed +TOTAL : 0.678822 sec + 1,562,866,097 cycles:u # 2.287 GHz + 3,160,253,003 instructions:u # 2.02 insn per cycle + 0.686387174 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2629) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.avx2_f_inl0/runTest.exe @@ -110,13 +132,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.137933e+06 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.137933e+06 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.141415e+06 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.141415e+06 ) sec^-1 MeanMatrixElemValue = ( 2.085720e+00 +- 4.835915e-03 ) GeV^0 -TOTAL : 0.638720 sec - 1,471,997,982 cycles:u # 2.288 GHz - 3,050,674,328 instructions:u # 2.07 insn per cycle - 0.646338395 seconds time elapsed +TOTAL : 0.639742 sec + 1,475,827,469 cycles:u # 2.289 GHz + 3,050,674,421 instructions:u # 2.07 insn per cycle + 0.647492887 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2501) (512y: 26) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512y_f_inl0/runTest.exe @@ -126,13 +148,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[16] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 8.184860e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 8.184860e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 8.180507e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 8.180507e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085721e+00 +- 4.835915e-03 ) GeV^0 -TOTAL : 0.818624 sec - 1,432,989,702 cycles:u # 1.742 GHz - 2,177,381,973 instructions:u # 1.52 insn per cycle - 0.825819338 seconds time elapsed +TOTAL : 0.818541 sec + 1,432,879,398 cycles:u # 1.740 GHz + 2,177,381,942 instructions:u # 1.52 insn per cycle + 0.827001096 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1706) (512y: 12) (512z: 1779) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512z_f_inl0/runTest.exe diff --git a/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_f_inl1.txt b/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_f_inl1.txt index 345b301771..40495f3b2f 100644 --- a/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_f_inl1.txt +++ b/epochX/cudacpp/tput/logs_ggtt_auto/log_ggtt_auto_f_inl1.txt @@ -1,59 +1,81 @@ /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx OMPFLAGS= +AVX=512y +FPTYPE=f +HELINL=1 +RNDGEN=curdev +Building in BUILDDIR=build.512y_f_inl1 for tag=512y_f_inl1_curdev (USEBUILDDIR is set = 1) + +make USEBUILDDIR=1 AVX=none +OMPFLAGS= AVX=none FPTYPE=f HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.none_f_inl1 for tag=none_f_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.none_f_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make[1]: Nothing to be done for `all.none_f_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make USEBUILDDIR=1 AVX=sse4 OMPFLAGS= AVX=sse4 FPTYPE=f HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.sse4_f_inl1 for tag=sse4_f_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.sse4_f_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make[1]: Nothing to be done for `all.sse4_f_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make USEBUILDDIR=1 AVX=avx2 OMPFLAGS= AVX=avx2 FPTYPE=f HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.avx2_f_inl1 for tag=avx2_f_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.avx2_f_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make[1]: Nothing to be done for `all.avx2_f_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make USEBUILDDIR=1 AVX=512y OMPFLAGS= AVX=512y FPTYPE=f HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.512y_f_inl1 for tag=512y_f_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512y_f_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make[1]: Nothing to be done for `all.512y_f_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make USEBUILDDIR=1 AVX=512z OMPFLAGS= AVX=512z FPTYPE=f HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.512z_f_inl1 for tag=512z_f_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512z_f_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' +make[1]: Nothing to be done for `all.512z_f_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx' -DATE: 2021-12-01_10:49:40 +DATE: 2021-12-01_22:14:27 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.none_f_inl1/gcheck.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTX_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 3.378138e+08 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.803138e+08 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 3.335690e+08 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.852243e+08 ) sec^-1 MeanMatrixElemValue = ( 2.085652e+00 +- 4.835323e-03 ) GeV^0 -TOTAL : 0.831964 sec - 739,027,153 cycles:u # 0.740 GHz - 1,483,617,609 instructions:u # 2.01 insn per cycle - 1.113943483 seconds time elapsed +TOTAL : 0.830514 sec + 746,466,125 cycles:u # 0.731 GHz + 1,484,741,028 instructions:u # 1.99 insn per cycle + 1.127767054 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 96 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... @@ -62,13 +84,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 2.492472e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 2.492472e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 2.492999e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 2.492999e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085710e+00 +- 4.835783e-03 ) GeV^0 -TOTAL : 2.284811 sec - 6,054,859,307 cycles:u # 2.646 GHz - 16,170,560,643 instructions:u # 2.67 insn per cycle - 2.292112704 seconds time elapsed +TOTAL : 2.286134 sec + 6,052,715,608 cycles:u # 2.644 GHz + 16,170,560,649 instructions:u # 2.67 insn per cycle + 2.293600614 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 682) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.none_f_inl1/runTest.exe @@ -78,13 +100,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 5.691714e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 5.691714e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 5.688856e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 5.688856e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085711e+00 +- 4.835793e-03 ) GeV^0 -TOTAL : 1.104665 sec - 2,903,922,806 cycles:u # 2.618 GHz - 6,525,917,384 instructions:u # 2.25 insn per cycle - 1.111912784 seconds time elapsed +TOTAL : 1.106839 sec + 2,906,068,809 cycles:u # 2.615 GHz + 6,525,916,720 instructions:u # 2.25 insn per cycle + 1.114390571 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 2785) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.sse4_f_inl1/runTest.exe @@ -94,13 +116,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 7.402418e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 7.402418e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 7.425149e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 7.425149e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085720e+00 +- 4.835915e-03 ) GeV^0 -TOTAL : 0.883470 sec - 2,031,553,377 cycles:u # 2.289 GHz - 3,704,033,940 instructions:u # 1.82 insn per cycle - 0.890330929 seconds time elapsed +TOTAL : 0.885333 sec + 2,033,417,945 cycles:u # 2.285 GHz + 3,704,035,593 instructions:u # 1.82 insn per cycle + 0.892693305 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 3503) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.avx2_f_inl1/runTest.exe @@ -110,13 +132,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 7.212886e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 7.212886e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 7.210509e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 7.210509e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085720e+00 +- 4.835915e-03 ) GeV^0 -TOTAL : 0.903901 sec - 2,076,333,677 cycles:u # 2.286 GHz - 3,578,727,346 instructions:u # 1.72 insn per cycle - 0.910968670 seconds time elapsed +TOTAL : 0.906197 sec + 2,079,001,172 cycles:u # 2.282 GHz + 3,578,727,713 instructions:u # 1.72 insn per cycle + 0.913784385 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 3338) (512y: 24) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512y_f_inl1/runTest.exe @@ -126,13 +148,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcess Process = SIGMA_SM_GG_TTX_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[16] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 5.695668e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 5.695668e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 5.692136e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 5.692136e+05 ) sec^-1 MeanMatrixElemValue = ( 2.085721e+00 +- 4.835915e-03 ) GeV^0 -TOTAL : 1.097831 sec - 1,872,089,939 cycles:u # 1.700 GHz - 3,068,718,694 instructions:u # 1.64 insn per cycle - 1.104959937 seconds time elapsed +TOTAL : 1.099868 sec + 1,875,843,490 cycles:u # 1.699 GHz + 3,068,718,508 instructions:u # 1.64 insn per cycle + 1.107606659 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 3364) (512y: 0) (512z: 1959) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_tt.auto/SubProcesses/P1_Sigma_sm_gg_ttx/build.512z_f_inl1/runTest.exe From 9d335ef6f54f7bf8ff464ea5b6b583ab7de587f7 Mon Sep 17 00:00:00 2001 From: Andrea Valassi Date: Wed, 1 Dec 2021 22:32:40 +0100 Subject: [PATCH 33/34] [klas5X] rerun ggttgg auto tests - performance seems exactly the same within fluctuations up and down --- .../log_ggttgg_auto_d_inl0.txt | 118 +++++++++++------- .../log_ggttgg_auto_d_inl1.txt | 118 +++++++++++------- .../log_ggttgg_auto_f_inl0.txt | 118 +++++++++++------- .../log_ggttgg_auto_f_inl1.txt | 118 +++++++++++------- 4 files changed, 280 insertions(+), 192 deletions(-) diff --git a/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_d_inl0.txt b/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_d_inl0.txt index e1e3466467..6a248bbab6 100644 --- a/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_d_inl0.txt +++ b/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_d_inl0.txt @@ -1,84 +1,106 @@ /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg OMPFLAGS= +AVX=512y +FPTYPE=d +HELINL=0 +RNDGEN=curdev +Building in BUILDDIR=build.512y_d_inl0 for tag=512y_d_inl0_curdev (USEBUILDDIR is set = 1) + +make USEBUILDDIR=1 AVX=none +OMPFLAGS= AVX=none FPTYPE=d HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.none_d_inl0 for tag=none_d_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.none_d_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.none_d_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=sse4 OMPFLAGS= AVX=sse4 FPTYPE=d HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.sse4_d_inl0 for tag=sse4_d_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.sse4_d_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.sse4_d_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=avx2 OMPFLAGS= AVX=avx2 FPTYPE=d HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.avx2_d_inl0 for tag=avx2_d_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.avx2_d_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.avx2_d_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=512y OMPFLAGS= AVX=512y FPTYPE=d HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.512y_d_inl0 for tag=512y_d_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512y_d_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.512y_d_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=512z OMPFLAGS= AVX=512z FPTYPE=d HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.512z_d_inl0 for tag=512z_d_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512z_d_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.512z_d_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' -DATE: 2021-12-01_11:33:25 +DATE: 2021-12-01_22:24:35 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl0/gcheck.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 4.388543e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.391508e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.394505e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.397606e+05 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 5.406831 sec - 12,424,319,754 cycles:u # 2.322 GHz - 23,282,976,350 instructions:u # 1.87 insn per cycle - 5.697459970 seconds time elapsed +TOTAL : 5.885811 sec + 12,887,512,331 cycles:u # 2.207 GHz + 23,283,886,611 instructions:u # 1.81 insn per cycle + 6.187895981 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 255 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl0/gcheck.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 4.886991e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.888706e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.885636e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.888022e+05 ) sec^-1 MeanMatrixElemValue = ( 6.665112e+00 +- 5.002651e+00 ) GeV^-4 -TOTAL : 2.717279 sec - 2,120,203,860 cycles:u # 0.718 GHz - 4,508,397,222 instructions:u # 2.13 insn per cycle - 3.016722705 seconds time elapsed +TOTAL : 2.725243 sec + 2,134,865,083 cycles:u # 0.719 GHz + 4,359,722,069 instructions:u # 2.04 insn per cycle + 3.030011688 seconds time elapsed ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl0/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.829510e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.829510e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.828777e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.828777e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 8.985019 sec - 23,984,698,394 cycles:u # 2.669 GHz - 74,429,676,608 instructions:u # 3.10 insn per cycle - 8.991662009 seconds time elapsed +TOTAL : 8.988190 sec + 23,987,052,314 cycles:u # 2.668 GHz + 74,429,675,717 instructions:u # 3.10 insn per cycle + 8.994748866 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 1470) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl0/runTest.exe @@ -88,13 +110,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProce Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[2] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 3.376446e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.376446e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 3.373852e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.373852e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 4.877238 sec - 13,014,333,492 cycles:u # 2.667 GHz - 39,610,810,845 instructions:u # 3.04 insn per cycle - 4.883926335 seconds time elapsed +TOTAL : 4.882944 sec + 13,022,783,729 cycles:u # 2.665 GHz + 39,610,811,029 instructions:u # 3.04 insn per cycle + 4.889644200 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 9180) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_d_inl0/runTest.exe @@ -104,13 +126,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProce Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 6.885384e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 6.885384e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 6.894147e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 6.894147e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 2.402076 sec - 5,451,716,257 cycles:u # 2.266 GHz - 13,654,534,285 instructions:u # 2.50 insn per cycle - 2.408731848 seconds time elapsed +TOTAL : 2.402283 sec + 5,449,527,130 cycles:u # 2.267 GHz + 13,654,534,249 instructions:u # 2.51 insn per cycle + 2.408948940 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 7734) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_d_inl0/runTest.exe @@ -120,13 +142,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProce Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 7.684325e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 7.684325e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 7.695323e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 7.695323e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 2.153827 sec - 4,888,081,054 cycles:u # 2.265 GHz - 12,420,337,101 instructions:u # 2.54 insn per cycle - 2.161066835 seconds time elapsed +TOTAL : 2.151631 sec + 4,886,821,638 cycles:u # 2.268 GHz + 12,420,337,242 instructions:u # 2.54 insn per cycle + 2.158562679 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 7385) (512y: 54) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_d_inl0/runTest.exe @@ -136,13 +158,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProce Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 6.512901e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 6.512901e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 6.524904e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 6.524904e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 2.538882 sec - 4,004,155,725 cycles:u # 1.575 GHz - 6,320,992,069 instructions:u # 1.58 insn per cycle - 2.545872732 seconds time elapsed +TOTAL : 2.534224 sec + 3,999,991,990 cycles:u # 1.576 GHz + 6,320,992,262 instructions:u # 1.58 insn per cycle + 2.540934497 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1904) (512y: 72) (512z: 5995) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_d_inl0/runTest.exe diff --git a/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_d_inl1.txt b/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_d_inl1.txt index cde118c374..b5691632f8 100644 --- a/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_d_inl1.txt +++ b/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_d_inl1.txt @@ -1,84 +1,106 @@ /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg OMPFLAGS= +AVX=512y +FPTYPE=d +HELINL=1 +RNDGEN=curdev +Building in BUILDDIR=build.512y_d_inl1 for tag=512y_d_inl1_curdev (USEBUILDDIR is set = 1) + +make USEBUILDDIR=1 AVX=none +OMPFLAGS= AVX=none FPTYPE=d HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.none_d_inl1 for tag=none_d_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.none_d_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.none_d_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=sse4 OMPFLAGS= AVX=sse4 FPTYPE=d HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.sse4_d_inl1 for tag=sse4_d_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.sse4_d_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.sse4_d_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=avx2 OMPFLAGS= AVX=avx2 FPTYPE=d HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.avx2_d_inl1 for tag=avx2_d_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.avx2_d_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.avx2_d_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=512y OMPFLAGS= AVX=512y FPTYPE=d HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.512y_d_inl1 for tag=512y_d_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512y_d_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.512y_d_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=512z OMPFLAGS= AVX=512z FPTYPE=d HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.512z_d_inl1 for tag=512z_d_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512z_d_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.512z_d_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' -DATE: 2021-12-01_11:34:08 +DATE: 2021-12-01_22:25:39 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl1/gcheck.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 2.743772e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 2.745221e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 2.740383e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 2.741850e+05 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 8.800843 sec - 20,514,940,846 cycles:u # 2.334 GHz - 39,330,070,250 instructions:u # 1.92 insn per cycle - 9.099733830 seconds time elapsed +TOTAL : 8.875098 sec + 21,000,723,811 cycles:u # 2.335 GHz + 39,320,253,434 instructions:u # 1.87 insn per cycle + 9.183863290 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 255 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl1/gcheck.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 3.233892e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.234663e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 3.230482e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.231513e+05 ) sec^-1 MeanMatrixElemValue = ( 6.665112e+00 +- 5.002651e+00 ) GeV^-4 -TOTAL : 3.802337 sec - 3,252,305,204 cycles:u # 0.802 GHz - 6,391,528,355 instructions:u # 1.97 insn per cycle - 4.114623733 seconds time elapsed +TOTAL : 3.817904 sec + 3,212,734,507 cycles:u # 0.788 GHz + 6,657,532,901 instructions:u # 2.07 insn per cycle + 4.136163260 seconds time elapsed ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl1/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.675128e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.675128e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.671969e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.671969e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 9.811676 sec - 26,193,775,743 cycles:u # 2.669 GHz - 64,879,362,711 instructions:u # 2.48 insn per cycle - 9.818873852 seconds time elapsed +TOTAL : 9.830345 sec + 26,199,174,062 cycles:u # 2.664 GHz + 64,879,363,603 instructions:u # 2.48 insn per cycle + 9.837420880 seconds time elapsed =Symbols in CPPProcess.o= (~sse4:13340) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl1/runTest.exe @@ -88,13 +110,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProce Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[2] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 2.660372e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 2.660372e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 2.662168e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 2.662168e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 6.184729 sec - 16,501,089,303 cycles:u # 2.667 GHz - 39,698,251,771 instructions:u # 2.41 insn per cycle - 6.191328560 seconds time elapsed +TOTAL : 6.185007 sec + 16,502,719,567 cycles:u # 2.668 GHz + 39,698,252,489 instructions:u # 2.41 insn per cycle + 6.191736715 seconds time elapsed =Symbols in CPPProcess.o= (~sse4:68925) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_d_inl1/runTest.exe @@ -104,13 +126,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProce Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 6.168104e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 6.168104e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 6.171026e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 6.171026e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 2.680501 sec - 6,078,293,584 cycles:u # 2.265 GHz - 13,044,180,861 instructions:u # 2.15 insn per cycle - 2.687213411 seconds time elapsed +TOTAL : 2.678248 sec + 6,077,914,495 cycles:u # 2.267 GHz + 13,044,180,305 instructions:u # 2.15 insn per cycle + 2.685304844 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2:45306) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_d_inl1/runTest.exe @@ -120,13 +142,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProce Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 5.677635e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 5.677635e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 5.678877e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 5.678877e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 2.908723 sec - 6,601,374,610 cycles:u # 2.267 GHz - 12,188,498,566 instructions:u # 1.85 insn per cycle - 2.915453886 seconds time elapsed +TOTAL : 2.908481 sec + 6,604,578,903 cycles:u # 2.268 GHz + 12,188,497,792 instructions:u # 1.85 insn per cycle + 2.915332495 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2:41857) (512y: 215) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_d_inl1/runTest.exe @@ -136,13 +158,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProce Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 7.254946e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 7.254946e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 7.264651e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 7.264651e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 2.282559 sec - 3,594,194,675 cycles:u # 1.572 GHz - 5,990,655,876 instructions:u # 1.67 insn per cycle - 2.289318334 seconds time elapsed +TOTAL : 2.277777 sec + 3,592,514,482 cycles:u # 1.575 GHz + 5,990,655,459 instructions:u # 1.67 insn per cycle + 2.284455459 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1966) (512y: 308) (512z:38918) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_d_inl1/runTest.exe diff --git a/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_f_inl0.txt b/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_f_inl0.txt index 0251bbd4ff..d764e7e65f 100644 --- a/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_f_inl0.txt +++ b/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_f_inl0.txt @@ -1,84 +1,106 @@ /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg OMPFLAGS= +AVX=512y +FPTYPE=f +HELINL=0 +RNDGEN=curdev +Building in BUILDDIR=build.512y_f_inl0 for tag=512y_f_inl0_curdev (USEBUILDDIR is set = 1) + +make USEBUILDDIR=1 AVX=none +OMPFLAGS= AVX=none FPTYPE=f HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.none_f_inl0 for tag=none_f_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.none_f_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.none_f_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=sse4 OMPFLAGS= AVX=sse4 FPTYPE=f HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.sse4_f_inl0 for tag=sse4_f_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.sse4_f_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.sse4_f_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=avx2 OMPFLAGS= AVX=avx2 FPTYPE=f HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.avx2_f_inl0 for tag=avx2_f_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.avx2_f_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.avx2_f_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=512y OMPFLAGS= AVX=512y FPTYPE=f HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.512y_f_inl0 for tag=512y_f_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512y_f_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.512y_f_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=512z OMPFLAGS= AVX=512z FPTYPE=f HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.512z_f_inl0 for tag=512z_f_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512z_f_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.512z_f_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' -DATE: 2021-12-01_11:35:03 +DATE: 2021-12-01_22:27:05 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl0/gcheck.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 7.199621e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 7.204622e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 7.054826e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 7.061169e+05 ) sec^-1 MeanMatrixElemValue = ( 4.059596e+00 +- 2.368053e+00 ) GeV^-4 -TOTAL : 4.858525 sec - 10,369,103,655 cycles:u # 2.154 GHz - 20,824,177,445 instructions:u # 2.01 insn per cycle - 5.138932722 seconds time elapsed +TOTAL : 5.051012 sec + 10,899,323,643 cycles:u # 2.164 GHz + 20,824,364,848 instructions:u # 1.91 insn per cycle + 5.334080479 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 255 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl0/gcheck.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 9.282397e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 9.285858e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 9.244001e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 9.248314e+05 ) sec^-1 MeanMatrixElemValue = ( 6.664703e+00 +- 5.072736e+00 ) GeV^-4 -TOTAL : 1.736571 sec - 1,237,189,827 cycles:u # 0.632 GHz - 2,496,769,062 instructions:u # 2.02 insn per cycle - 2.023754293 seconds time elapsed +TOTAL : 1.752006 sec + 1,263,078,949 cycles:u # 0.637 GHz + 2,493,687,816 instructions:u # 1.97 insn per cycle + 2.042658988 seconds time elapsed ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl0/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.803847e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.803847e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.804549e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.804549e+03 ) sec^-1 MeanMatrixElemValue = ( 4.059968e+00 +- 2.367799e+00 ) GeV^-4 -TOTAL : 9.108182 sec - 24,321,116,779 cycles:u # 2.669 GHz - 73,388,316,467 instructions:u # 3.02 insn per cycle - 9.115275885 seconds time elapsed +TOTAL : 9.108249 sec + 24,321,336,071 cycles:u # 2.670 GHz + 73,388,316,545 instructions:u # 3.02 insn per cycle + 9.114972183 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 1303) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl0/runTest.exe @@ -88,13 +110,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProce Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 6.581704e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 6.581704e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 6.587866e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 6.587866e+03 ) sec^-1 MeanMatrixElemValue = ( 4.060121e+00 +- 2.367902e+00 ) GeV^-4 -TOTAL : 2.508219 sec - 6,691,545,763 cycles:u # 2.664 GHz - 20,504,413,107 instructions:u # 3.06 insn per cycle - 2.514656367 seconds time elapsed +TOTAL : 2.507075 sec + 6,688,774,265 cycles:u # 2.665 GHz + 20,504,413,749 instructions:u # 3.07 insn per cycle + 2.513707902 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 9974) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_f_inl0/runTest.exe @@ -104,13 +126,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProce Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.314398e+04 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.314398e+04 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.315227e+04 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.315227e+04 ) sec^-1 MeanMatrixElemValue = ( 4.059418e+00 +- 2.367906e+00 ) GeV^-4 -TOTAL : 1.264337 sec - 2,868,425,059 cycles:u # 2.262 GHz - 6,999,592,055 instructions:u # 2.44 insn per cycle - 1.270669627 seconds time elapsed +TOTAL : 1.263719 sec + 2,866,082,017 cycles:u # 2.260 GHz + 6,999,591,871 instructions:u # 2.44 insn per cycle + 1.270387908 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 8722) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_f_inl0/runTest.exe @@ -120,13 +142,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProce Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.414232e+04 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.414232e+04 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.403850e+04 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.403850e+04 ) sec^-1 MeanMatrixElemValue = ( 4.059418e+00 +- 2.367906e+00 ) GeV^-4 -TOTAL : 1.176112 sec - 2,669,787,249 cycles:u # 2.263 GHz - 6,472,447,344 instructions:u # 2.42 insn per cycle - 1.182393727 seconds time elapsed +TOTAL : 1.184815 sec + 2,688,490,791 cycles:u # 2.263 GHz + 6,472,447,312 instructions:u # 2.41 insn per cycle + 1.191774631 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 8308) (512y: 26) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_f_inl0/runTest.exe @@ -136,13 +158,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProce Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[16] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.316884e+04 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.316884e+04 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.321081e+04 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.321081e+04 ) sec^-1 MeanMatrixElemValue = ( 4.059419e+00 +- 2.367907e+00 ) GeV^-4 -TOTAL : 1.262979 sec - 1,992,609,610 cycles:u # 1.574 GHz - 3,310,550,374 instructions:u # 1.66 insn per cycle - 1.270179286 seconds time elapsed +TOTAL : 1.258250 sec + 1,986,759,074 cycles:u # 1.575 GHz + 3,310,550,130 instructions:u # 1.67 insn per cycle + 1.264852919 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2919) (512y: 12) (512z: 6122) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_f_inl0/runTest.exe diff --git a/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_f_inl1.txt b/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_f_inl1.txt index 4abb804da2..f43baaab2f 100644 --- a/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_f_inl1.txt +++ b/epochX/cudacpp/tput/logs_ggttgg_auto/log_ggttgg_auto_f_inl1.txt @@ -1,84 +1,106 @@ /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg OMPFLAGS= +AVX=512y +FPTYPE=f +HELINL=1 +RNDGEN=curdev +Building in BUILDDIR=build.512y_f_inl1 for tag=512y_f_inl1_curdev (USEBUILDDIR is set = 1) + +make USEBUILDDIR=1 AVX=none +OMPFLAGS= AVX=none FPTYPE=f HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.none_f_inl1 for tag=none_f_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.none_f_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.none_f_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=sse4 OMPFLAGS= AVX=sse4 FPTYPE=f HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.sse4_f_inl1 for tag=sse4_f_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.sse4_f_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.sse4_f_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=avx2 OMPFLAGS= AVX=avx2 FPTYPE=f HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.avx2_f_inl1 for tag=avx2_f_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.avx2_f_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.avx2_f_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=512y OMPFLAGS= AVX=512y FPTYPE=f HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.512y_f_inl1 for tag=512y_f_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512y_f_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.512y_f_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=512z OMPFLAGS= AVX=512z FPTYPE=f HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.512z_f_inl1 for tag=512z_f_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512z_f_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.512z_f_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' -DATE: 2021-12-01_11:35:38 +DATE: 2021-12-01_22:27:59 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl1/gcheck.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 6.698727e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 6.703795e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 6.678742e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 6.683963e+05 ) sec^-1 MeanMatrixElemValue = ( 4.059596e+00 +- 2.368053e+00 ) GeV^-4 -TOTAL : 36.741951 sec - 94,895,472,348 cycles:u # 2.579 GHz - 96,390,235,215 instructions:u # 1.02 insn per cycle - 37.042689612 seconds time elapsed +TOTAL : 38.055340 sec + 98,618,305,460 cycles:u # 2.582 GHz + 96,387,100,604 instructions:u # 0.98 insn per cycle + 38.352729053 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 255 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl1/gcheck.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 8.539080e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 8.541783e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 8.539219e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 8.542922e+05 ) sec^-1 MeanMatrixElemValue = ( 6.664703e+00 +- 5.072736e+00 ) GeV^-4 -TOTAL : 1.809388 sec - 1,319,182,354 cycles:u # 0.647 GHz - 2,675,253,742 instructions:u # 2.03 insn per cycle - 2.102071784 seconds time elapsed +TOTAL : 1.812573 sec + 1,296,333,004 cycles:u # 0.633 GHz + 2,668,627,799 instructions:u # 2.06 insn per cycle + 2.107843721 seconds time elapsed ========================================================================= runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl1/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.765018e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.765018e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.765031e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.765031e+03 ) sec^-1 MeanMatrixElemValue = ( 4.059968e+00 +- 2.367799e+00 ) GeV^-4 -TOTAL : 9.309502 sec - 24,858,794,988 cycles:u # 2.669 GHz - 64,759,551,750 instructions:u # 2.61 insn per cycle - 9.316350169 seconds time elapsed +TOTAL : 9.310280 sec + 24,864,352,871 cycles:u # 2.670 GHz + 64,759,552,216 instructions:u # 2.60 insn per cycle + 9.316819802 seconds time elapsed =Symbols in CPPProcess.o= (~sse4:13244) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl1/runTest.exe @@ -88,13 +110,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProce Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 6.017002e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 6.017002e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 6.019451e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 6.019451e+03 ) sec^-1 MeanMatrixElemValue = ( 4.059968e+00 +- 2.367799e+00 ) GeV^-4 -TOTAL : 2.742261 sec - 7,314,780,966 cycles:u # 2.664 GHz - 20,197,668,479 instructions:u # 2.76 insn per cycle - 2.748555485 seconds time elapsed +TOTAL : 2.741385 sec + 7,314,092,406 cycles:u # 2.664 GHz + 20,197,668,515 instructions:u # 2.76 insn per cycle + 2.747933332 seconds time elapsed =Symbols in CPPProcess.o= (~sse4:69323) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_f_inl1/runTest.exe @@ -104,13 +126,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProce Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.190092e+04 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.190092e+04 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.189419e+04 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.189419e+04 ) sec^-1 MeanMatrixElemValue = ( 4.058922e+00 +- 2.367605e+00 ) GeV^-4 -TOTAL : 1.395964 sec - 3,161,843,033 cycles:u # 2.261 GHz - 6,774,004,748 instructions:u # 2.14 insn per cycle - 1.402268598 seconds time elapsed +TOTAL : 1.402107 sec + 3,162,648,737 cycles:u # 2.256 GHz + 6,774,006,073 instructions:u # 2.14 insn per cycle + 1.408865168 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2:47130) (512y: 0) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_f_inl1/runTest.exe @@ -120,13 +142,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProce Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.109812e+04 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.109812e+04 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.110731e+04 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.110731e+04 ) sec^-1 MeanMatrixElemValue = ( 4.058922e+00 +- 2.367605e+00 ) GeV^-4 -TOTAL : 1.494450 sec - 3,388,366,334 cycles:u # 2.262 GHz - 6,266,409,846 instructions:u # 1.85 insn per cycle - 1.500646579 seconds time elapsed +TOTAL : 1.495464 sec + 3,387,719,348 cycles:u # 2.262 GHz + 6,266,409,354 instructions:u # 1.85 insn per cycle + 1.502240557 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2:43251) (512y: 31) (512z: 0) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_f_inl1/runTest.exe @@ -136,13 +158,13 @@ runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProce Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[16] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.407201e+04 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.407201e+04 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.406444e+04 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.406444e+04 ) sec^-1 MeanMatrixElemValue = ( 4.058923e+00 +- 2.367605e+00 ) GeV^-4 -TOTAL : 1.182769 sec - 1,867,605,024 cycles:u # 1.575 GHz - 3,179,037,328 instructions:u # 1.70 insn per cycle - 1.189243237 seconds time elapsed +TOTAL : 1.183206 sec + 1,867,305,961 cycles:u # 1.574 GHz + 3,179,037,685 instructions:u # 1.70 insn per cycle + 1.190022504 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 0) (avx2: 4162) (512y: 7) (512z:39589) ------------------------------------------------------------------------- runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_f_inl1/runTest.exe From a4733d43fad41d959ec4925e251e7d9f15f6e798 Mon Sep 17 00:00:00 2001 From: Andrea Valassi Date: Wed, 1 Dec 2021 22:34:59 +0100 Subject: [PATCH 34/34] [klas5X] *** COMPLETE KLAS5X *** copy logs ./tput/copyLogs.sh -a2m -ggttgg --- .../log_ggttgg_manu_d_inl0.txt | 152 ++++++++++-------- .../log_ggttgg_manu_d_inl1.txt | 152 ++++++++++-------- .../log_ggttgg_manu_f_inl0.txt | 152 ++++++++++-------- .../log_ggttgg_manu_f_inl1.txt | 152 ++++++++++-------- 4 files changed, 348 insertions(+), 260 deletions(-) diff --git a/epochX/cudacpp/tput/logs_ggttgg_manu/log_ggttgg_manu_d_inl0.txt b/epochX/cudacpp/tput/logs_ggttgg_manu/log_ggttgg_manu_d_inl0.txt index cfbe2c76ae..6a248bbab6 100644 --- a/epochX/cudacpp/tput/logs_ggttgg_manu/log_ggttgg_manu_d_inl0.txt +++ b/epochX/cudacpp/tput/logs_ggttgg_manu/log_ggttgg_manu_d_inl0.txt @@ -1,151 +1,173 @@ -/afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg +/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg +OMPFLAGS= +AVX=512y +FPTYPE=d +HELINL=0 +RNDGEN=curdev +Building in BUILDDIR=build.512y_d_inl0 for tag=512y_d_inl0_curdev (USEBUILDDIR is set = 1) + +make USEBUILDDIR=1 AVX=none OMPFLAGS= AVX=none FPTYPE=d HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.none_d_inl0 for tag=none_d_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.none_d_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.none_d_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=sse4 OMPFLAGS= AVX=sse4 FPTYPE=d HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.sse4_d_inl0 for tag=sse4_d_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.sse4_d_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.sse4_d_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=avx2 OMPFLAGS= AVX=avx2 FPTYPE=d HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.avx2_d_inl0 for tag=avx2_d_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.avx2_d_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.avx2_d_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=512y OMPFLAGS= AVX=512y FPTYPE=d HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.512y_d_inl0 for tag=512y_d_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512y_d_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.512y_d_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=512z OMPFLAGS= AVX=512z FPTYPE=d HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.512z_d_inl0 for tag=512z_d_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512z_d_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.512z_d_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' -DATE: 2021-10-30_18:32:35 +DATE: 2021-12-01_22:24:35 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl0/gcheck.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl0/gcheck.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 4.389065e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.392199e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.394505e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.397606e+05 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 5.527304 sec - 12,338,636,544 cycles:u # 2.195 GHz - 23,284,301,818 instructions:u # 1.89 insn per cycle - 5.818163976 seconds time elapsed +TOTAL : 5.885811 sec + 12,887,512,331 cycles:u # 2.207 GHz + 23,283,886,611 instructions:u # 1.81 insn per cycle + 6.187895981 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 255 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl0/gcheck.exe -p 2048 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl0/gcheck.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 4.888569e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 4.890197e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 4.885636e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 4.888022e+05 ) sec^-1 MeanMatrixElemValue = ( 6.665112e+00 +- 5.002651e+00 ) GeV^-4 -TOTAL : 2.715465 sec - 2,207,888,800 cycles:u # 0.748 GHz - 4,315,288,646 instructions:u # 1.95 insn per cycle - 3.014457509 seconds time elapsed +TOTAL : 2.725243 sec + 2,134,865,083 cycles:u # 0.719 GHz + 4,359,722,069 instructions:u # 2.04 insn per cycle + 3.030011688 seconds time elapsed ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl0/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl0/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.835255e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.835255e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.828777e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.828777e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 8.963365 sec - 23,915,001,762 cycles:u # 2.668 GHz - 74,429,667,913 instructions:u # 3.11 insn per cycle - 8.970744124 seconds time elapsed +TOTAL : 8.988190 sec + 23,987,052,314 cycles:u # 2.668 GHz + 74,429,675,717 instructions:u # 3.10 insn per cycle + 8.994748866 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 1470) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl0/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_d_inl0/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_d_inl0/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[2] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 3.360002e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.360002e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 3.373852e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.373852e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 4.900822 sec - 13,074,422,521 cycles:u # 2.666 GHz - 39,630,346,835 instructions:u # 3.03 insn per cycle - 4.907845467 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 9242) (avx2: 0) (512y: 0) (512z: 0) +TOTAL : 4.882944 sec + 13,022,783,729 cycles:u # 2.665 GHz + 39,610,811,029 instructions:u # 3.04 insn per cycle + 4.889644200 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 9180) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_d_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_d_inl0/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_d_inl0/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_d_inl0/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 6.850812e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 6.850812e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 6.894147e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 6.894147e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 2.413604 sec - 5,474,520,871 cycles:u # 2.265 GHz - 13,651,272,052 instructions:u # 2.49 insn per cycle - 2.433877016 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 7704) (512y: 0) (512z: 0) +TOTAL : 2.402283 sec + 5,449,527,130 cycles:u # 2.267 GHz + 13,654,534,249 instructions:u # 2.51 insn per cycle + 2.408948940 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 7734) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_d_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_d_inl0/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_d_inl0/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_d_inl0/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 7.679431e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 7.679431e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 7.695323e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 7.695323e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 2.155346 sec - 4,892,023,660 cycles:u # 2.265 GHz - 12,417,075,197 instructions:u # 2.54 insn per cycle - 2.178706120 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 7358) (512y: 61) (512z: 0) +TOTAL : 2.151631 sec + 4,886,821,638 cycles:u # 2.268 GHz + 12,420,337,242 instructions:u # 2.54 insn per cycle + 2.158562679 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 7385) (512y: 54) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_d_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_d_inl0/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_d_inl0/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_d_inl0/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 6.520577e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 6.520577e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 6.524904e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 6.524904e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 2.536400 sec - 4,000,932,863 cycles:u # 1.575 GHz - 6,319,705,721 instructions:u # 1.58 insn per cycle - 2.543150860 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1836) (512y: 74) (512z: 6010) +TOTAL : 2.534224 sec + 3,999,991,990 cycles:u # 1.576 GHz + 6,320,992,262 instructions:u # 1.58 insn per cycle + 2.540934497 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1904) (512y: 72) (512z: 5995) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_d_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_d_inl0/runTest.exe [ PASSED ] 3 tests. ========================================================================= diff --git a/epochX/cudacpp/tput/logs_ggttgg_manu/log_ggttgg_manu_d_inl1.txt b/epochX/cudacpp/tput/logs_ggttgg_manu/log_ggttgg_manu_d_inl1.txt index 5c09240986..b5691632f8 100644 --- a/epochX/cudacpp/tput/logs_ggttgg_manu/log_ggttgg_manu_d_inl1.txt +++ b/epochX/cudacpp/tput/logs_ggttgg_manu/log_ggttgg_manu_d_inl1.txt @@ -1,151 +1,173 @@ -/afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg +/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg +OMPFLAGS= +AVX=512y +FPTYPE=d +HELINL=1 +RNDGEN=curdev +Building in BUILDDIR=build.512y_d_inl1 for tag=512y_d_inl1_curdev (USEBUILDDIR is set = 1) + +make USEBUILDDIR=1 AVX=none OMPFLAGS= AVX=none FPTYPE=d HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.none_d_inl1 for tag=none_d_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.none_d_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.none_d_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=sse4 OMPFLAGS= AVX=sse4 FPTYPE=d HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.sse4_d_inl1 for tag=sse4_d_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.sse4_d_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.sse4_d_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=avx2 OMPFLAGS= AVX=avx2 FPTYPE=d HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.avx2_d_inl1 for tag=avx2_d_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.avx2_d_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.avx2_d_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=512y OMPFLAGS= AVX=512y FPTYPE=d HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.512y_d_inl1 for tag=512y_d_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512y_d_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.512y_d_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=512z OMPFLAGS= AVX=512z FPTYPE=d HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.512z_d_inl1 for tag=512z_d_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512z_d_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.512z_d_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' -DATE: 2021-10-30_18:33:23 +DATE: 2021-12-01_22:25:39 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl1/gcheck.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl1/gcheck.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 2.759865e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 2.761403e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 2.740383e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 2.741850e+05 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 8.810741 sec - 20,498,814,252 cycles:u # 2.306 GHz - 39,312,269,575 instructions:u # 1.92 insn per cycle - 9.110551643 seconds time elapsed +TOTAL : 8.875098 sec + 21,000,723,811 cycles:u # 2.335 GHz + 39,320,253,434 instructions:u # 1.87 insn per cycle + 9.183863290 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 255 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl1/gcheck.exe -p 2048 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl1/gcheck.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 3.235137e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 3.235859e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 3.230482e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 3.231513e+05 ) sec^-1 MeanMatrixElemValue = ( 6.665112e+00 +- 5.002651e+00 ) GeV^-4 -TOTAL : 3.812562 sec - 3,204,986,045 cycles:u # 0.788 GHz - 6,495,315,451 instructions:u # 2.03 insn per cycle - 4.129323118 seconds time elapsed +TOTAL : 3.817904 sec + 3,212,734,507 cycles:u # 0.788 GHz + 6,657,532,901 instructions:u # 2.07 insn per cycle + 4.136163260 seconds time elapsed ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl1/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl1/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.670840e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.670840e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.671969e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.671969e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 9.837905 sec - 26,230,415,720 cycles:u # 2.665 GHz - 64,879,355,634 instructions:u # 2.47 insn per cycle - 9.845568668 seconds time elapsed +TOTAL : 9.830345 sec + 26,199,174,062 cycles:u # 2.664 GHz + 64,879,363,603 instructions:u # 2.48 insn per cycle + 9.837420880 seconds time elapsed =Symbols in CPPProcess.o= (~sse4:13340) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_d_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_d_inl1/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_d_inl1/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[2] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 2.669194e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 2.669194e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 2.662168e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 2.662168e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 6.170961 sec - 16,444,547,696 cycles:u # 2.665 GHz - 39,259,113,269 instructions:u # 2.39 insn per cycle - 6.177884594 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4:68136) (avx2: 0) (512y: 0) (512z: 0) +TOTAL : 6.185007 sec + 16,502,719,567 cycles:u # 2.668 GHz + 39,698,252,489 instructions:u # 2.41 insn per cycle + 6.191736715 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4:68925) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_d_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_d_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_d_inl1/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_d_inl1/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 6.149470e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 6.149470e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 6.171026e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 6.171026e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 2.687638 sec - 6,097,620,613 cycles:u # 2.266 GHz - 13,071,882,358 instructions:u # 2.14 insn per cycle - 2.694649751 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2:45381) (512y: 0) (512z: 0) +TOTAL : 2.678248 sec + 6,077,914,495 cycles:u # 2.267 GHz + 13,044,180,305 instructions:u # 2.15 insn per cycle + 2.685304844 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2:45306) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_d_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_d_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_d_inl1/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_d_inl1/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 5.663119e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 5.663119e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 5.678877e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 5.678877e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 2.915918 sec - 6,617,502,580 cycles:u # 2.266 GHz - 12,168,442,885 instructions:u # 1.84 insn per cycle - 2.922933773 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2:41747) (512y: 222) (512z: 0) +TOTAL : 2.908481 sec + 6,604,578,903 cycles:u # 2.268 GHz + 12,188,497,792 instructions:u # 1.85 insn per cycle + 2.915332495 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2:41857) (512y: 215) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_d_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_d_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_d_inl1/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_d_inl1/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = DOUBLE (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 7.242604e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 7.242604e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 7.264651e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 7.264651e+03 ) sec^-1 MeanMatrixElemValue = ( 4.063123e+00 +- 2.368970e+00 ) GeV^-4 -TOTAL : 2.285556 sec - 3,598,507,830 cycles:u # 1.572 GHz - 5,965,707,680 instructions:u # 1.66 insn per cycle - 2.292362801 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1920) (512y: 310) (512z:38736) +TOTAL : 2.277777 sec + 3,592,514,482 cycles:u # 1.575 GHz + 5,990,655,459 instructions:u # 1.67 insn per cycle + 2.284455459 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 1966) (512y: 308) (512z:38918) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_d_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_d_inl1/runTest.exe [ PASSED ] 3 tests. ========================================================================= diff --git a/epochX/cudacpp/tput/logs_ggttgg_manu/log_ggttgg_manu_f_inl0.txt b/epochX/cudacpp/tput/logs_ggttgg_manu/log_ggttgg_manu_f_inl0.txt index ba6fb8c999..d764e7e65f 100644 --- a/epochX/cudacpp/tput/logs_ggttgg_manu/log_ggttgg_manu_f_inl0.txt +++ b/epochX/cudacpp/tput/logs_ggttgg_manu/log_ggttgg_manu_f_inl0.txt @@ -1,151 +1,173 @@ -/afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg +/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg +OMPFLAGS= +AVX=512y +FPTYPE=f +HELINL=0 +RNDGEN=curdev +Building in BUILDDIR=build.512y_f_inl0 for tag=512y_f_inl0_curdev (USEBUILDDIR is set = 1) + +make USEBUILDDIR=1 AVX=none OMPFLAGS= AVX=none FPTYPE=f HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.none_f_inl0 for tag=none_f_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.none_f_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.none_f_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=sse4 OMPFLAGS= AVX=sse4 FPTYPE=f HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.sse4_f_inl0 for tag=sse4_f_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.sse4_f_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.sse4_f_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=avx2 OMPFLAGS= AVX=avx2 FPTYPE=f HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.avx2_f_inl0 for tag=avx2_f_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.avx2_f_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.avx2_f_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=512y OMPFLAGS= AVX=512y FPTYPE=f HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.512y_f_inl0 for tag=512y_f_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512y_f_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.512y_f_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=512z OMPFLAGS= AVX=512z FPTYPE=f HELINL=0 RNDGEN=curdev Building in BUILDDIR=build.512z_f_inl0 for tag=512z_f_inl0_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512z_f_inl0_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.512z_f_inl0_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' -DATE: 2021-10-30_18:34:21 +DATE: 2021-12-01_22:27:05 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl0/gcheck.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl0/gcheck.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 7.073155e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 7.078433e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 7.054826e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 7.061169e+05 ) sec^-1 MeanMatrixElemValue = ( 4.059596e+00 +- 2.368053e+00 ) GeV^-4 -TOTAL : 4.785720 sec - 10,542,358,937 cycles:u # 2.152 GHz - 20,825,640,660 instructions:u # 1.98 insn per cycle - 5.070025129 seconds time elapsed +TOTAL : 5.051012 sec + 10,899,323,643 cycles:u # 2.164 GHz + 20,824,364,848 instructions:u # 1.91 insn per cycle + 5.334080479 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 255 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl0/gcheck.exe -p 2048 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl0/gcheck.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 9.262137e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 9.265269e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 9.244001e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 9.248314e+05 ) sec^-1 MeanMatrixElemValue = ( 6.664703e+00 +- 5.072736e+00 ) GeV^-4 -TOTAL : 1.739678 sec - 1,221,029,381 cycles:u # 0.620 GHz - 2,587,413,449 instructions:u # 2.12 insn per cycle - 2.029921431 seconds time elapsed +TOTAL : 1.752006 sec + 1,263,078,949 cycles:u # 0.637 GHz + 2,493,687,816 instructions:u # 1.97 insn per cycle + 2.042658988 seconds time elapsed ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl0/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl0/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.804640e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.804640e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.804549e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.804549e+03 ) sec^-1 MeanMatrixElemValue = ( 4.059968e+00 +- 2.367799e+00 ) GeV^-4 -TOTAL : 9.105290 sec - 24,314,441,109 cycles:u # 2.670 GHz - 73,388,307,046 instructions:u # 3.02 insn per cycle - 9.111816577 seconds time elapsed +TOTAL : 9.108249 sec + 24,321,336,071 cycles:u # 2.670 GHz + 73,388,316,545 instructions:u # 3.02 insn per cycle + 9.114972183 seconds time elapsed =Symbols in CPPProcess.o= (~sse4: 1303) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl0/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_f_inl0/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_f_inl0/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 6.595718e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 6.595718e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 6.587866e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 6.587866e+03 ) sec^-1 MeanMatrixElemValue = ( 4.060121e+00 +- 2.367902e+00 ) GeV^-4 -TOTAL : 2.503318 sec - 6,676,322,655 cycles:u # 2.663 GHz - 20,497,987,828 instructions:u # 3.07 insn per cycle - 2.510098184 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 9985) (avx2: 0) (512y: 0) (512z: 0) +TOTAL : 2.507075 sec + 6,688,774,265 cycles:u # 2.665 GHz + 20,504,413,749 instructions:u # 3.07 insn per cycle + 2.513707902 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 9974) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_f_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_f_inl0/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_f_inl0/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_f_inl0/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.312741e+04 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.312741e+04 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.315227e+04 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.315227e+04 ) sec^-1 MeanMatrixElemValue = ( 4.059418e+00 +- 2.367906e+00 ) GeV^-4 -TOTAL : 1.267228 sec - 2,872,370,000 cycles:u # 2.261 GHz - 6,998,826,703 instructions:u # 2.44 insn per cycle - 1.273952513 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 8680) (512y: 0) (512z: 0) +TOTAL : 1.263719 sec + 2,866,082,017 cycles:u # 2.260 GHz + 6,999,591,871 instructions:u # 2.44 insn per cycle + 1.270387908 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 8722) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_f_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_f_inl0/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_f_inl0/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_f_inl0/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.412353e+04 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.412353e+04 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.403850e+04 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.403850e+04 ) sec^-1 MeanMatrixElemValue = ( 4.059418e+00 +- 2.367906e+00 ) GeV^-4 -TOTAL : 1.177859 sec - 2,669,991,031 cycles:u # 2.259 GHz - 6,471,289,460 instructions:u # 2.42 insn per cycle - 1.185265584 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 8259) (512y: 28) (512z: 0) +TOTAL : 1.184815 sec + 2,688,490,791 cycles:u # 2.263 GHz + 6,472,447,312 instructions:u # 2.41 insn per cycle + 1.191774631 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 8308) (512y: 26) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_f_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_f_inl0/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_f_inl0/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_f_inl0/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=0] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[16] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.321058e+04 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.321058e+04 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.321081e+04 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.321081e+04 ) sec^-1 MeanMatrixElemValue = ( 4.059419e+00 +- 2.367907e+00 ) GeV^-4 -TOTAL : 1.258439 sec - 1,987,629,690 cycles:u # 1.575 GHz - 3,310,248,582 instructions:u # 1.67 insn per cycle - 1.264989274 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2823) (512y: 13) (512z: 6139) +TOTAL : 1.258250 sec + 1,986,759,074 cycles:u # 1.575 GHz + 3,310,550,130 instructions:u # 1.67 insn per cycle + 1.264852919 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 2919) (512y: 12) (512z: 6122) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_f_inl0/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_f_inl0/runTest.exe [ PASSED ] 3 tests. ========================================================================= diff --git a/epochX/cudacpp/tput/logs_ggttgg_manu/log_ggttgg_manu_f_inl1.txt b/epochX/cudacpp/tput/logs_ggttgg_manu/log_ggttgg_manu_f_inl1.txt index b1050678ef..f43baaab2f 100644 --- a/epochX/cudacpp/tput/logs_ggttgg_manu/log_ggttgg_manu_f_inl1.txt +++ b/epochX/cudacpp/tput/logs_ggttgg_manu/log_ggttgg_manu_f_inl1.txt @@ -1,151 +1,173 @@ -/afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg +/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg +OMPFLAGS= +AVX=512y +FPTYPE=f +HELINL=1 +RNDGEN=curdev +Building in BUILDDIR=build.512y_f_inl1 for tag=512y_f_inl1_curdev (USEBUILDDIR is set = 1) + +make USEBUILDDIR=1 AVX=none OMPFLAGS= AVX=none FPTYPE=f HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.none_f_inl1 for tag=none_f_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.none_f_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.none_f_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=sse4 OMPFLAGS= AVX=sse4 FPTYPE=f HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.sse4_f_inl1 for tag=sse4_f_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.sse4_f_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.sse4_f_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=avx2 OMPFLAGS= AVX=avx2 FPTYPE=f HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.avx2_f_inl1 for tag=avx2_f_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.avx2_f_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.avx2_f_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=512y OMPFLAGS= AVX=512y FPTYPE=f HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.512y_f_inl1 for tag=512y_f_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512y_f_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.512y_f_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make USEBUILDDIR=1 AVX=512z OMPFLAGS= AVX=512z FPTYPE=f HELINL=1 RNDGEN=curdev Building in BUILDDIR=build.512z_f_inl1 for tag=512z_f_inl1_curdev (USEBUILDDIR is set = 1) -make: Nothing to be done for `all.512z_f_inl1_curdev'. +make[1]: Entering directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' +make[1]: Nothing to be done for `all.512z_f_inl1_curdev'. +make[1]: Leaving directory `/data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg' -DATE: 2021-10-30_18:35:00 +DATE: 2021-12-01_22:27:59 On itscrd70.cern.ch [CPU: Intel(R) Xeon(R) Silver 4216 CPU] [GPU: 1x Tesla V100S-PCIE-32GB]: ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl1/gcheck.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl1/gcheck.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 6.755177e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 6.760594e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 6.678742e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 6.683963e+05 ) sec^-1 MeanMatrixElemValue = ( 4.059596e+00 +- 2.368053e+00 ) GeV^-4 -TOTAL : 36.365639 sec - 94,130,351,005 cycles:u # 2.576 GHz - 96,390,605,169 instructions:u # 1.02 insn per cycle - 36.669519084 seconds time elapsed +TOTAL : 38.055340 sec + 98,618,305,460 cycles:u # 2.582 GHz + 96,387,100,604 instructions:u # 0.98 insn per cycle + 38.352729053 seconds time elapsed ==PROF== Profiling "sigmaKin": launch__registers_per_thread 255 ==PROF== Profiling "sigmaKin": sm__sass_average_branch_targets_threads_uniform.pct 100% ......................................................................... -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl1/gcheck.exe -p 2048 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl1/gcheck.exe -p 2048 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CUDA [nvcc 11.1.105 (gcc 10.2.0)] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) -EvtsPerSec[MatrixElems] (3) = ( 8.538574e+05 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 8.541988e+05 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 8.539219e+05 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 8.542922e+05 ) sec^-1 MeanMatrixElemValue = ( 6.664703e+00 +- 5.072736e+00 ) GeV^-4 -TOTAL : 1.816556 sec - 1,293,676,002 cycles:u # 0.630 GHz - 2,662,594,694 instructions:u # 2.06 insn per cycle - 2.113360747 seconds time elapsed +TOTAL : 1.812573 sec + 1,296,333,004 cycles:u # 0.633 GHz + 2,668,627,799 instructions:u # 2.06 insn per cycle + 2.107843721 seconds time elapsed ========================================================================= -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl1/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl1/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = SCALAR ('none': ~vector[1], no SIMD) -EvtsPerSec[MatrixElems] (3) = ( 1.763046e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.763046e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.765031e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.765031e+03 ) sec^-1 MeanMatrixElemValue = ( 4.059968e+00 +- 2.367799e+00 ) GeV^-4 -TOTAL : 9.322550 sec - 24,873,516,803 cycles:u # 2.667 GHz - 64,759,544,405 instructions:u # 2.60 insn per cycle - 9.330907974 seconds time elapsed +TOTAL : 9.310280 sec + 24,864,352,871 cycles:u # 2.670 GHz + 64,759,552,216 instructions:u # 2.60 insn per cycle + 9.316819802 seconds time elapsed =Symbols in CPPProcess.o= (~sse4:13244) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.none_f_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_f_inl1/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_f_inl1/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[4] ('sse4': SSE4.2, 128bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 6.074995e+03 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 6.074995e+03 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 6.019451e+03 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 6.019451e+03 ) sec^-1 MeanMatrixElemValue = ( 4.059968e+00 +- 2.367799e+00 ) GeV^-4 -TOTAL : 2.716565 sec - 7,237,758,016 cycles:u # 2.661 GHz - 20,113,715,638 instructions:u # 2.78 insn per cycle - 2.723174385 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4:69018) (avx2: 0) (512y: 0) (512z: 0) +TOTAL : 2.741385 sec + 7,314,092,406 cycles:u # 2.664 GHz + 20,197,668,515 instructions:u # 2.76 insn per cycle + 2.747933332 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4:69323) (avx2: 0) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_f_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.sse4_f_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_f_inl1/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_f_inl1/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('avx2': AVX2, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.190222e+04 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.190222e+04 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.189419e+04 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.189419e+04 ) sec^-1 MeanMatrixElemValue = ( 4.058922e+00 +- 2.367605e+00 ) GeV^-4 -TOTAL : 1.395121 sec - 3,158,910,056 cycles:u # 2.258 GHz - 6,769,306,523 instructions:u # 2.14 insn per cycle - 1.401670810 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2:47060) (512y: 0) (512z: 0) +TOTAL : 1.402107 sec + 3,162,648,737 cycles:u # 2.256 GHz + 6,774,006,073 instructions:u # 2.14 insn per cycle + 1.408865168 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2:47130) (512y: 0) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_f_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.avx2_f_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_f_inl1/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_f_inl1/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[8] ('512y': AVX512, 256bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.098709e+04 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.098709e+04 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.110731e+04 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.110731e+04 ) sec^-1 MeanMatrixElemValue = ( 4.058922e+00 +- 2.367605e+00 ) GeV^-4 -TOTAL : 1.509692 sec - 3,422,149,013 cycles:u # 2.261 GHz - 6,279,290,687 instructions:u # 1.83 insn per cycle - 1.516283502 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2:43307) (512y: 32) (512z: 0) +TOTAL : 1.495464 sec + 3,387,719,348 cycles:u # 2.262 GHz + 6,266,409,354 instructions:u # 1.85 insn per cycle + 1.502240557 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2:43251) (512y: 31) (512z: 0) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_f_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512y_f_inl1/runTest.exe [ PASSED ] 3 tests. ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_f_inl1/check.exe -p 64 256 1 OMP= +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_f_inl1/check.exe -p 64 256 1 OMP= Process = SIGMA_SM_GG_TTXGG_CPP [gcc 10.2.0] [inlineHel=1] FP precision = FLOAT (NaN/abnormal=0, zero=0) Internal loops fptype_sv = VECTOR[16] ('512z': AVX512, 512bit) [cxtype_ref=YES] -EvtsPerSec[MatrixElems] (3) = ( 1.401341e+04 ) sec^-1 -EvtsPerSec[MECalcOnly] (3a) = ( 1.401341e+04 ) sec^-1 +EvtsPerSec[MatrixElems] (3) = ( 1.406444e+04 ) sec^-1 +EvtsPerSec[MECalcOnly] (3a) = ( 1.406444e+04 ) sec^-1 MeanMatrixElemValue = ( 4.058923e+00 +- 2.367605e+00 ) GeV^-4 -TOTAL : 1.187543 sec - 1,871,757,256 cycles:u # 1.571 GHz - 3,177,704,632 instructions:u # 1.70 insn per cycle - 1.194067694 seconds time elapsed -=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 4094) (512y: 7) (512z:39573) +TOTAL : 1.183206 sec + 1,867,305,961 cycles:u # 1.574 GHz + 3,179,037,685 instructions:u # 1.70 insn per cycle + 1.190022504 seconds time elapsed +=Symbols in CPPProcess.o= (~sse4: 0) (avx2: 4162) (512y: 7) (512z:39589) ------------------------------------------------------------------------- -runExe /afs/cern.ch/user/a/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_f_inl1/runTest.exe +runExe /data/avalassi/GPU2020/madgraph4gpuX/epochX/cudacpp/gg_ttgg.auto/SubProcesses/P1_Sigma_sm_gg_ttxgg/build.512z_f_inl1/runTest.exe [ PASSED ] 3 tests. =========================================================================