Remove artifacts support & fix rocSPARSE bug (#674)

pxl-th · web-flow · commit 84524f6494e8 · 2024-09-21T01:10:27.000+03:00
diff --git a/Project.toml b/Project.toml
@@ -11,7 +11,6 @@ CEnum = "fa961155-64e5-5f13-b03f-caf6b980ea82"
 ExprTools = "e2ba6199-217a-4e67-a87a-7c52f15ade04"
 GPUArrays = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7"
 GPUCompiler = "61eb1bfa-7361-4325-ad38-22787b887f55"
-HIP_jll = "2696aab5-0948-5276-aa9a-2a86a37016b8"
 KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c"
 LLD_jll = "d55e3150-da41-5e91-b323-ecfd1eec6109"
 LLVM = "929cbde3-209d-540e-8aea-75f648917ca0"
@@ -32,7 +31,6 @@ StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 UnsafeAtomics = "013be700-e6cd-48c3-b4a1-df204f14c38f"
 UnsafeAtomicsLLVM = "d80eeb9a-aca5-4d75-85e5-170c8b632249"
-hsa_rocr_jll = "dd59ff1a-a01a-568d-8b29-0669330f116a"
 
 [compat]
 AbstractFFTs = "1.0"
@@ -42,7 +40,6 @@ CEnum = "0.4, 0.5"
 ExprTools = "0.1"
 GPUArrays = "10"
 GPUCompiler = "0.26, 0.27"
-HIP_jll = "5.4"
 KernelAbstractions = "0.9.2"
 LLD_jll = "15, 16, 17"
 LLVM = "8.1"
@@ -56,5 +53,4 @@ SpecialFunctions = "2"
 StaticArraysCore = "1"
 UnsafeAtomics = "0.2"
 UnsafeAtomicsLLVM = "0.2"
-hsa_rocr_jll = "5.4"
 julia = "1.10"
diff --git a/docs/src/index.md b/docs/src/index.md
@@ -69,18 +69,6 @@ Standard path:
 If you have non-standard path for ROCm, set `ROCM_PATH=<path>`
 environment variable before launching Julia.
 
-## ROCm artifacts
-
-There is limited support for ROCm 5.4+ artifacts which can be enabled with
-[`AMDGPU.use_artifacts!`](@ref).
-
-Limited means not all libraries are available and some of the functionality
-may be disabled.
-
-```@docs
-AMDGPU.use_artifacts!
-```
-
 ## Extra Setup Details
 
 List of additional steps that may be needed to take to ensure everything is working:
@@ -139,9 +127,6 @@ Template of `LocalPreferences.toml` with all options:
 # If `true` (default), eagerly run GC to keep the pool from growing too big.
 # GC is triggered during new allocatoins or synchronization points.
 eager_gc = false
-# If `true` then use ROCm libraries provided by artifacts.
-# However, not all ROCm libraries are available as artifacts.
-use_artifacts = false
 # Use non-blocking synchronization for all `AMDGPU.synchronize()` calls.
 nonblocking_synchronization = true
 # Memory limit specifies maximum amount of memory in percentages
diff --git a/docs/src/profiling.md b/docs/src/profiling.md
@@ -69,6 +69,5 @@ wall duration is lower.
 
 ## Debugging
 
-Use `JULIA_AMDGPU_LAUNCH_BLOCKING=1` and `HIP_LAUNCH_BLOCKING=1` to
-synchronize immediately after launching GPU kernels.
+Use `HIP_LAUNCH_BLOCKING=1` to synchronize immediately after launching GPU kernels.
 This will allow to pinpoint exact kernel that caused the exception.
diff --git a/src/AMDGPU.jl b/src/AMDGPU.jl
@@ -156,31 +156,12 @@ function __init__()
         Runtime.RT_EXITING[] = true
     end
 
-    if haskey(ENV, "JULIA_AMDGPU_LAUNCH_BLOCKING")
-        launch_blocking = parse(Bool, ENV["JULIA_AMDGPU_LAUNCH_BLOCKING"])
-        LAUNCH_BLOCKING[] = launch_blocking
-        if launch_blocking
-            @info "`JULIA_AMDGPU_LAUNCH_BLOCKING` is set to `true`. " *
-                "Synchronizing immediately after every Julia kernel launch."
-        end
-    end
-
     if haskey(ENV, "HIP_LAUNCH_BLOCKING")
         launch_blocking = parse(Bool, ENV["HIP_LAUNCH_BLOCKING"])
+        LAUNCH_BLOCKING[] = launch_blocking
         if launch_blocking
-            @info "`HIP_AMDGPU_LAUNCH_BLOCKING` is set to `true`. " *
-                "Synchronizing immediately after every HIP kernel launch."
-        end
-    end
-
-    if haskey(ENV, "JULIA_AMDGPU_DISABLE_ARTIFACTS")
-        env_use_artifacts = !parse(Bool, ENV["JULIA_AMDGPU_DISABLE_ARTIFACTS"])
-        if use_artifacts() != env_use_artifacts
-            enable_artifacts!(env_use_artifacts)
-            @warn """
-            The environment variable JULIA_AMDGPU_DISABLE_ARTIFACTS does not match the value from preferences.
-            Forcing the preferences value to $(env_use_artifacts); please restart Julia for changes to take effect.
-            """
+            @info "`HIP_LAUNCH_BLOCKING` is set to `true`. " *
+                "Synchronizing immediately after every Julia & HIP kernel launch."
         end
     end
 
diff --git a/src/discovery/discovery.jl b/src/discovery/discovery.jl
@@ -6,56 +6,14 @@ using Libdl
 
 include("utils.jl")
 
-"""
-    use_artifacts!(flag::Bool = true)
-
-Pass `true` to switch from system-wide ROCm installtion to artifacts.
-When using artifacts, system-wide installation is not needed at all.
-"""
-function use_artifacts!(flag::Bool = true; show_message::Bool = true)
-    if flag && Base.libllvm_version >= v"16"
-        error("No supported artifacts for LLVM 16+. See: https://github.com/JuliaGPU/AMDGPU.jl/issues/440.")
-    end
-    @set_preferences!("use_artifacts" => flag)
-    if show_message
-        @info """
-        Switched `use_artifacts` to `$flag`.
-        Restart Julia session for the changes to take effect.
-        """
-    end
-end
-
-use_artifacts()::Bool = @load_preference("use_artifacts", false)
-
-@static if use_artifacts()
-    import hsa_rocr_jll
-end
-
-if haskey(ENV, "JULIA_AMDGPU_DISABLE_ARTIFACTS")
-    disable_artifacts = parse(Bool, get(ENV, "JULIA_AMDGPU_DISABLE_ARTIFACTS", "true"))
-    if !disable_artifacts && Base.libllvm_version >= v"16"
-        error("No supported artifacts for LLVM 16+. See: https://github.com/JuliaGPU/AMDGPU.jl/issues/440.")
-    end
-    use_artifacts!(!disable_artifacts; show_message=false)
-end
-
 function get_artifact_library(pkg::Symbol, libname::Symbol)::String
     succ, res = safe_exec("import $pkg; println($pkg.$libname)")
     (succ && ispath(res)) || return ""
     return res
 end
 
-function get_library(
-    libname::String; rocm_path::String,
-    artifact_library::Symbol, artifact_field::Union{Nothing, Symbol} = nothing,
-    ext::String = dlext,
-)
-    if use_artifacts()
-        artifact_field = artifact_field ≡ nothing ? Symbol(libname) : artifact_field
-        get_artifact_library(artifact_library, artifact_field)
-    else
-        find_rocm_library(libname, rocm_path, ext)
-    end
+function get_library(libname::String; rocm_path::String, ext::String = dlext)
+    find_rocm_library(libname, rocm_path, ext)
 end
 
 function get_ld_lld(rocm_path::String;
@@ -76,7 +34,6 @@ function get_device_libs(from_artifact::Bool; rocm_path::String)
     end
 end
 
-export use_artifacts, use_artifacts!
 export lld_artifact, lld_path, libhsaruntime, libdevice_libs, libhip
 export librocblas, librocsparse, librocsolver, librocalution
 export librocrand, librocfft, libMIOpen_path
@@ -108,14 +65,12 @@ function __init__()
         end
     end
 
-    rocm_path = use_artifacts() ? "" : find_roc_path()
+    rocm_path = find_roc_path()
     lib_prefix = Sys.islinux() ? "lib" : ""
 
     try
         global libhsaruntime = if Sys.islinux()
-            get_library("libhsa-runtime64";
-                    rocm_path, artifact_library=:hsa_rocr_jll,
-                    artifact_field=:libhsa_runtime64, ext="so.1")
+            get_library("libhsa-runtime64"; rocm_path, ext="so.1")
         else
             ""
         end
@@ -134,37 +89,26 @@ function __init__()
 
         # HIP.
         global libhip = get_library(
-            Sys.islinux() ? "libamdhip64" : "amdhip64";
-            rocm_path, artifact_library=:HIP_jll)
+            Sys.islinux() ? "libamdhip64" : "amdhip64"; rocm_path)
+
+        # Detect HIP version, which will influence what device libraries to use.
+        hip_version = Base.thisminor(_hip_runtime_version())
+        # Check if opaque pointers are enabled and turn off artifacts.
+        llvm_args = get(ENV, "JULIA_LLVM_ARGS", "")
+        enabled_opaque_pointers = occursin("-opaque-pointers", llvm_args)
+        from_artifact = hip_version > v"5.4" && !enabled_opaque_pointers
 
-        from_artifact = if isempty(libhip)
-            use_artifacts()
-        else
-            # Detect HIP version, which will influence what device libraries to use.
-            hip_version = Base.thisminor(_hip_runtime_version())
-            # Check if opaque pointers are enabled and turn off artifacts.
-            llvm_args = get(ENV, "JULIA_LLVM_ARGS", "")
-            enabled_opaque_pointers = occursin("-opaque-pointers", llvm_args)
-            (hip_version > v"5.4" && !enabled_opaque_pointers) ? true : use_artifacts()
-        end
         # If ROCm 5.5+ - use artifact device libraries.
         global libdevice_libs = get_device_libs(from_artifact; rocm_path)
 
         # HIP-based libraries.
-        global librocblas = get_library(lib_prefix * "rocblas";
-            rocm_path, artifact_library=:rocBLAS_jll)
-        global librocsparse = get_library(lib_prefix * "rocsparse";
-            rocm_path, artifact_library=:rocSPARSE_jll)
-        global librocsolver = get_library(lib_prefix * "rocsolver";
-            rocm_path, artifact_library=:rocSOLVER_jll)
-        global librocalution = get_library(lib_prefix * "rocalution";
-            rocm_path, artifact_library=:rocALUTION_jll)
-        global librocrand = get_library(lib_prefix * "rocrand";
-            rocm_path, artifact_library=:rocRAND_jll)
-        global librocfft = get_library(lib_prefix * "rocfft";
-            rocm_path, artifact_library=:rocFFT_jll)
-        global libMIOpen_path = get_library(lib_prefix * "MIOpen";
-            rocm_path, artifact_library=:MIOpen_jll)
+        global librocblas = get_library(lib_prefix * "rocblas"; rocm_path)
+        global librocsparse = get_library(lib_prefix * "rocsparse"; rocm_path)
+        global librocsolver = get_library(lib_prefix * "rocsolver"; rocm_path)
+        global librocalution = get_library(lib_prefix * "rocalution"; rocm_path)
+        global librocrand = get_library(lib_prefix * "rocrand"; rocm_path)
+        global librocfft = get_library(lib_prefix * "rocfft"; rocm_path)
+        global libMIOpen_path = get_library(lib_prefix * "MIOpen"; rocm_path)
     catch err
         @error """ROCm discovery failed!
         Discovered ROCm path: $rocm_path.
diff --git a/src/sparse/level2.jl b/src/sparse/level2.jl
@@ -184,7 +184,7 @@ for (bname,aname,sname,elty) in (
     (:rocsparse_scsrsv_buffer_size, :rocsparse_scsrsv_analysis, :rocsparse_scsrsv_solve, :Float32),
     (:rocsparse_dcsrsv_buffer_size, :rocsparse_dcsrsv_analysis, :rocsparse_dcsrsv_solve, :Float64),
     (:rocsparse_ccsrsv_buffer_size, :rocsparse_ccsrsv_analysis, :rocsparse_ccsrsv_solve, :ComplexF32),
-    (:rocsparse_ccsrsv_buffer_size, :rocsparse_zcsrsv_analysis, :rocsparse_zcsrsv_solve, :ComplexF64),
+    (:rocsparse_zcsrsv_buffer_size, :rocsparse_zcsrsv_analysis, :rocsparse_zcsrsv_solve, :ComplexF64),
 )
     @eval begin
         function sv2!(
@@ -230,8 +230,10 @@ for (bname,aname,sname,elty) in (
                     handle(), ctransa, m, nnz(A),
                     desc, nonzeros(A), A.colPtr, rowvals(A), info_ref[],
                     rocsparse_analysis_policy_force, rocsparse_solve_policy_auto, buffer)
+
                 posit = Ref{Cint}(1)
                 rocsparse_csrsv_zero_pivot(handle(), desc, info_ref[], posit)
+
                 if posit[] >= 0
                     rocsparse_destroy_mat_info(info_ref[])
                     error("Structural/numerical zero in A at ($(posit[]),$(posit[])))")
diff --git a/test/rocsparse/generic.jl b/test/rocsparse/generic.jl
@@ -117,10 +117,16 @@ for SparseMatrixType in (ROCSparseMatrixCSR, ROCSparseMatrixCSC, ROCSparseMatrix
                             dC = ROCArray(C)
                             alpha = rand(T)
                             rocSPARSE.sm!(transa, transb, uplo, diag, alpha, dA, dB, dC, 'O', algo)
-                            if transb == 'T'
-                                @test_broken opa(A) \ (alpha * opb(B)) ≈ collect(dC)
-                            else
+
+                            if AMDGPU.HIP.runtime_version() ≥ v"6.2"
                                 @test opa(A) \ (alpha * opb(B)) ≈ collect(dC)
+                            # TODO remove once we support only ROCm 6.2+
+                            else
+                                if transb == 'T'
+                                    @test_broken opa(A) \ (alpha * opb(B)) ≈ collect(dC)
+                                else
+                                    @test opa(A) \ (alpha * opb(B)) ≈ collect(dC)
+                                end
                             end
                         end
                     end
diff --git a/test/rocsparse/preconditioners.jl b/test/rocsparse/preconditioners.jl
@@ -35,7 +35,9 @@ end
                             T = uplo == 'L' ? tril(T) : triu(T)
                             T = diag == 'N' ? T : T - Diagonal(T) + I
                             T = sparse(T)
-                            d_T = SparseMatrixType == ROCSparseMatrixBSR ? SparseMatrixType(ROCSparseMatrixCSR(T), blockdim) : SparseMatrixType(T)
+                            d_T = SparseMatrixType == ROCSparseMatrixBSR ?
+                                SparseMatrixType(ROCSparseMatrixCSR(T), blockdim) :
+                                SparseMatrixType(T)
                             x = rand(elty,n)
                             d_x = ROCVector{elty}(x)
                             d_y = rocSPARSE.sv2(trans, uplo, diag, d_T, d_x, 'O')
@@ -58,7 +60,9 @@ end
                                 T = uplo == 'L' ? tril(T) : triu(T)
                                 T = diag == 'N' ? T : T - Diagonal(T) + I
                                 T = sparse(T)
-                                d_T = SparseMatrixType == ROCSparseMatrixBSR ? SparseMatrixType(ROCSparseMatrixCSR(T), blockdim) : SparseMatrixType(T)
+                                d_T = SparseMatrixType == ROCSparseMatrixBSR ?
+                                    SparseMatrixType(ROCSparseMatrixCSR(T), blockdim) :
+                                    SparseMatrixType(T)
                                 X = transX == 'N' ? rand(elty,n,p) : rand(elty,p,n)
                                 d_X = ROCMatrix{elty}(X)
                                 d_Y = rocSPARSE.sm2(transT, transX, uplo, diag, d_T, d_X, 'O')
