rocmPackages.rocprofiler: fix rocprof and rocprofv2

4 files changed, 181 insertions, 27 deletions

diff --git a/pkgs/development/rocm-modules/5/default.nix b/pkgs/development/rocm-modules/5/default.nix
index 3229d7b077a..d5f1c2678c4 100644
--- a/pkgs/development/rocm-modules/5/default.nix
+++ b/pkgs/development/rocm-modules/5/default.nix
@@ -105,8 +105,8 @@ in rec {
 
   # Needs GCC
   rocprofiler = callPackage ./rocprofiler {
+    inherit rocmUpdateScript clr rocm-core rocm-thunk rocm-device-libs roctracer rocdbgapi rocm-smi hsa-amd-aqlprofile-bin;
     inherit (llvm) clang;
-    inherit rocmUpdateScript clr rocm-thunk roctracer rocm-smi hsa-amd-aqlprofile-bin;
   };
 
   # Needs GCC
diff --git a/pkgs/development/rocm-modules/5/rocprofiler/0000-dont-install-tests-hsaco.patch b/pkgs/development/rocm-modules/5/rocprofiler/0000-dont-install-tests-hsaco.patch
new file mode 100644
index 00000000000..aaa31c4974d
--- /dev/null
+++ b/pkgs/development/rocm-modules/5/rocprofiler/0000-dont-install-tests-hsaco.patch
@@ -0,0 +1,15 @@
+diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
+index 8473a42..07ea873 100644
+--- a/test/CMakeLists.txt
++++ b/test/CMakeLists.txt
+@@ -112,10 +112,6 @@ function(generate_hsaco TARGET_ID INPUT_FILE OUTPUT_FILE)
+         DEPENDS ${INPUT_FILE} clang
+         COMMENT "Building ${OUTPUT_FILE}..."
+         VERBATIM)
+-    install(
+-        FILES ${PROJECT_BINARY_DIR}/${OUTPUT_FILE}
+-        DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}/tests-v1
+-        COMPONENT tests)
+     set(HSACO_TARGET_LIST
+         ${HSACO_TARGET_LIST} ${PROJECT_BINARY_DIR}/${OUTPUT_FILE}
+         PARENT_SCOPE)
diff --git a/pkgs/development/rocm-modules/5/rocprofiler/0001-fix-shell-scripts.patch b/pkgs/development/rocm-modules/5/rocprofiler/0001-fix-shell-scripts.patch
new file mode 100644
index 00000000000..aba7709c4ed
--- /dev/null
+++ b/pkgs/development/rocm-modules/5/rocprofiler/0001-fix-shell-scripts.patch
@@ -0,0 +1,111 @@
+diff --git a/bin/rocprofv2 b/bin/rocprofv2
+index 92f7489..8839d4d 100755
+--- a/bin/rocprofv2
++++ b/bin/rocprofv2
+@@ -2,7 +2,8 @@
+ set -eo pipefail
+ CURRENT_DIR="$(dirname -- "$0")"
+ ROCPROFV2_DIR=$(dirname -- $(realpath ${BASH_SOURCE[0]}))
+-ROCM_DIR=$(dirname -- "$ROCPROFV2_DIR")
++ROCPROFILER_DIR=$(dirname -- "$ROCPROFV2_DIR")
++ROCM_DIR=@rocmtoolkit_merged@
+ PLUGIN_LIST=("ctf" "perfetto" "file" "att")
+ RUN_FROM_BUILD=0
+ if [[ $ROCPROFV2_DIR == *"/build"* ]]; then
+@@ -10,7 +11,7 @@ if [[ $ROCPROFV2_DIR == *"/build"* ]]; then
+   ROCM_DIR=$ROCPROFV2_DIR
+ fi
+ 
+-export ROCPROFILER_METRICS_PATH=$ROCM_DIR/libexec/rocprofiler/counters/derived_counters.xml
++export ROCPROFILER_METRICS_PATH=$ROCPROFILER_DIR/libexec/rocprofiler/counters/derived_counters.xml
+ export LD_LIBRARY_PATH=$ROCM_DIR/lib:$LD_LIBRARY_PATH
+ 
+ usage() {
+@@ -70,8 +71,8 @@ while [ 1 ]; do
+       exit 1
+     fi
+   elif [[ "$1" = "--list-counters" ]]; then
+-    export LD_PRELOAD=$LD_PRELOAD:$ROCM_DIR/lib/rocprofiler/librocprofiler_tool.so
+-    eval $ROCM_DIR/libexec/rocprofiler/ctrl
++    export LD_PRELOAD=$LD_PRELOAD:$ROCPROFILER_DIR/lib/rocprofiler/librocprofiler_tool.so
++    eval $ROCPROFILER_DIR/libexec/rocprofiler/ctrl
+     exit 1
+   elif [[ "$1" = "-i" || "$1" = "--input" ]]; then
+     if [ $2 ] && [ -n $2 ] && [ -r $2 ]; then
+@@ -171,7 +172,7 @@ while [ 1 ]; do
+       if [ $RUN_FROM_BUILD == 1 ]; then
+         ATT_PATH=$ROCM_DIR/plugin/att/att/att.py
+       else
+-        ATT_PATH=$ROCM_DIR/libexec/rocprofiler/att/att.py
++        ATT_PATH=$ROCPROFILER_DIR/libexec/rocprofiler/att/att.py
+         export ROCPROFV2_ATT_LIB_PATH=$ROCM_DIR/lib/hsa-amd-aqlprofile/librocprofv2_att.so
+       fi
+       ATT_ARGV=$3
+@@ -236,13 +237,13 @@ if [ -n "$PMC_LINES" ] && [ ! -n "$ATT_ARGV" ]; then
+       export OUTPUT_PATH=$FINAL_PATH
+     fi
+     let COUNTER=COUNTER+1
+-    LD_PRELOAD=$LD_PRELOAD:$ROCM_DIR/lib/rocprofiler/librocprofiler_tool.so $*
++    LD_PRELOAD=$LD_PRELOAD:$ROCPROFILER_DIR/lib/rocprofiler/librocprofiler_tool.so $*
+     if [ -n "$OUTPUT_PATH" ]; then
+       echo -e "\nThe output path for the following counters: $OUTPUT_PATH"
+     fi
+   done
+ else
+-  LD_PRELOAD=$LD_PRELOAD:$ROCM_DIR/lib/rocprofiler/librocprofiler_tool.so $*
++  LD_PRELOAD=$LD_PRELOAD:$ROCPROFILER_DIR/lib/rocprofiler/librocprofiler_tool.so $*
+ fi
+ 
+ get_pmc_results_txt_path() {
+diff --git a/bin/rpl_run.sh b/bin/rpl_run.sh
+index c1a3daa..bc868a5 100755
+--- a/bin/rpl_run.sh
++++ b/bin/rpl_run.sh
+@@ -24,16 +24,17 @@
+ 
+ time_stamp=`date +%y%m%d_%H%M%S`
+ BIN_DIR=$(dirname $(realpath ${BASH_SOURCE[0]}))
+-ROOT_DIR=$(dirname $BIN_DIR)
++ROCPROFILER_DIR=$(dirname $BIN_DIR)
++ROOT_DIR=@rocmtoolkit_merged@
+ RUN_DIR=`pwd`
+ TMP_DIR="/tmp"
+ DATA_DIR="rpl_data_${time_stamp}_$$"
+ 
+-RPL_PATH=$ROOT_DIR/lib
++RPL_PATH=$ROCPROFILER_DIR/lib
+ TLIB_PATH=$RPL_PATH/rocprofiler
+ TTLIB_PATH=$ROOT_DIR/lib/roctracer
+ ROCM_LIB_PATH=$ROOT_DIR/lib
+-PROF_BIN_DIR=$ROOT_DIR/libexec/rocprofiler
++PROF_BIN_DIR=$ROCPROFILER_DIR/libexec/rocprofiler
+ 
+ if [ -z "$ROCP_PYTHON_VERSION" ] ; then
+   ROCP_PYTHON_VERSION=python3
+@@ -73,7 +74,7 @@ export ROCP_METRICS=$TLIB_PATH/metrics.xml
+ # Disable AQL-profile read API
+ export AQLPROFILE_READ_API=0
+ # ROC Profiler package path
+-export ROCP_PACKAGE_DIR=$ROOT_DIR
++export ROCP_PACKAGE_DIR=$ROCPROFILER_DIR
+ # enabled SPM KFD mode
+ export ROCP_SPM_KFD_MODE=1
+ 
+@@ -350,7 +351,7 @@ convert_time_val() {
+ 
+ ################################################################################################
+ # main
+-echo "RPL: on '$time_stamp' from '$ROOT_DIR' in '$RUN_DIR'"
++echo "RPL: on '$time_stamp' from '$ROCPROFILER_DIR' in '$RUN_DIR'"
+ # Parsing arguments
+ if [ -z "$1" ] ; then
+   usage
+@@ -557,7 +558,7 @@ elif [ "$input_type" = "txt" -o "$input_type" = "none" ] ; then
+   else
+     echo "<metric></metric>" > $RES_DIR/input.xml
+   fi
+-  input_list=`/bin/ls $RES_DIR/input*.xml`
++  input_list=`ls $RES_DIR/input*.xml`
+   export ROCPROFILER_SESS=$RES_DIR
+ else
+   fatal "Bad input file type '$INPUT_FILE'"
diff --git a/pkgs/development/rocm-modules/5/rocprofiler/default.nix b/pkgs/development/rocm-modules/5/rocprofiler/default.nix
index c77014b50cf..c66c56a64eb 100644
--- a/pkgs/development/rocm-modules/5/rocprofiler/default.nix
+++ b/pkgs/development/rocm-modules/5/rocprofiler/default.nix
@@ -2,11 +2,16 @@
 , stdenv
 , fetchFromGitHub
 , rocmUpdateScript
+, symlinkJoin
+, substituteAll
 , cmake
 , clang
 , clr
+, rocm-core
 , rocm-thunk
+, rocm-device-libs
 , roctracer
+, rocdbgapi
 , rocm-smi
 , hsa-amd-aqlprofile-bin
 , numactl
@@ -14,24 +19,32 @@
 , libxml2
 , elfutils
 , mpi
+, systemd
 , gtest
 , python3Packages
-, gpuTargets ? [
-  "gfx900"
-  "gfx906"
-  "gfx908"
-  "gfx90a"
-  "gfx940"
-  "gfx941"
-  "gfx942"
-  "gfx1030"
-  "gfx1100"
-  "gfx1101"
-  "gfx1102"
-]
+, gpuTargets ? clr.gpuTargets
 }:
 
-stdenv.mkDerivation (finalAttrs: {
+let
+  rocmtoolkit-merged = symlinkJoin {
+    name = "rocmtoolkit-merged";
+
+    paths = [
+      rocm-core
+      rocm-thunk
+      rocm-device-libs
+      roctracer
+      rocdbgapi
+      rocm-smi
+      hsa-amd-aqlprofile-bin
+      clr
+    ];
+
+    postBuild = ''
+      rm -rf $out/nix-support
+    '';
+  };
+in stdenv.mkDerivation (finalAttrs: {
   pname = "rocprofiler";
   version = "5.7.1";
 
@@ -42,6 +55,17 @@ stdenv.mkDerivation (finalAttrs: {
     hash = "sha256-1s/7C9y+73ADLF/17Vepw0pZNVtYnKoP24GdwKc9X2Y=";
   };
 
+  patches = [
+    # These just simply won't build
+    ./0000-dont-install-tests-hsaco.patch
+
+    # Fix bad paths
+    (substituteAll {
+      src = ./0001-fix-shell-scripts.patch;
+      rocmtoolkit_merged = rocmtoolkit-merged;
+    })
+  ];
+
   nativeBuildInputs = [
     cmake
     clang
@@ -53,20 +77,19 @@ stdenv.mkDerivation (finalAttrs: {
   ];
 
   buildInputs = [
-    rocm-thunk
-    rocm-smi
-    hsa-amd-aqlprofile-bin
     numactl
     libpciaccess
     libxml2
     elfutils
     mpi
+    systemd
     gtest
   ];
 
+  propagatedBuildInputs = [ rocmtoolkit-merged ];
+
   cmakeFlags = [
     "-DCMAKE_MODULE_PATH=${clr}/lib/cmake/hip"
-    "-DPROF_API_HEADER_PATH=${roctracer.src}/inc/ext"
     "-DHIP_ROOT_DIR=${clr}"
     "-DGPU_TARGETS=${lib.concatStringsSep ";" gpuTargets}"
     # Manually define CMAKE_INSTALL_<DIR>
@@ -79,16 +102,21 @@ stdenv.mkDerivation (finalAttrs: {
   postPatch = ''
     patchShebangs .
 
-    # Cannot find ROCm device library, pointless
-    substituteInPlace CMakeLists.txt \
-      --replace "add_subdirectory(tests-v2)" "" \
-      --replace "add_subdirectory(samples)" ""
+    substituteInPlace tests-v2/featuretests/profiler/CMakeLists.txt \
+      --replace "--build-id=sha1" "--build-id=sha1 --rocm-path=${clr} --rocm-device-lib-path=${rocm-device-libs}/amdgcn/bitcode"
+
+    substituteInPlace test/CMakeLists.txt \
+      --replace "\''${ROCM_ROOT_DIR}/amdgcn/bitcode" "${rocm-device-libs}/amdgcn/bitcode"
   '';
 
-  postBuild = ''
-    # HSACO aren't being built for some reason
-    substituteInPlace test/cmake_install.cmake \
-      --replace "file(INSTALL DESTINATION \"\''${CMAKE_INSTALL_PREFIX}/share/rocprofiler/tests-v1\" TYPE FILE FILES \"" "message(\""
+  postInstall = ''
+    # Why do these not already have the executable bit set?
+    chmod +x $out/lib/rocprofiler/librocprof-tool.so
+    chmod +x $out/share/rocprofiler/tests-v1/test/ocl/SimpleConvolution
+
+    # Why do these have the executable bit set?
+    chmod -x $out/libexec/rocprofiler/counters/basic_counters.xml
+    chmod -x $out/libexec/rocprofiler/counters/derived_counters.xml
   '';
 
   passthru.updateScript = rocmUpdateScript {