Merge pull request #40 from gridap/rhs_assembly

Rhs assembly alone

Author: fverdugo

Manifest.toml

@@ -10,38 +10,38 @@ uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
 
 [[ArrayInterface]]
 deps = ["Compat", "IfElse", "LinearAlgebra", "Requires", "SparseArrays", "Static"]
-git-tree-sha1 = "b8d49c34c3da35f220e7295659cd0bab8e739fed"
+git-tree-sha1 = "1d6835607e9f214cb4210310868f8cf07eb0facc"
 uuid = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
-version = "3.1.33"
+version = "3.1.34"
 
 [[ArrayLayouts]]
 deps = ["FillArrays", "LinearAlgebra", "SparseArrays"]
-git-tree-sha1 = "a745df3fdce8aff885e6f4a3b7427cc27ce5f86c"
+git-tree-sha1 = "7a92ea1dd16472d18ca1ffcbb7b3cc67d7e78a3f"
 uuid = "4c555306-a7a7-4459-81d9-ec55ddd5c99a"
-version = "0.7.6"
+version = "0.7.7"
 
 [[Artifacts]]
 uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
 
 [[BSON]]
-git-tree-sha1 = "92b8a8479128367aaab2620b8e73dff632f5ae69"
+git-tree-sha1 = "ebcd6e22d69f21249b7b8668351ebf42d6dc87a1"
 uuid = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0"
-version = "0.3.3"
+version = "0.3.4"
 
 [[Base64]]
 uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
 [[BlockArrays]]
 deps = ["ArrayLayouts", "FillArrays", "LinearAlgebra"]
-git-tree-sha1 = "782362509cf50a51092f513e92783c18ab0b6d51"
+git-tree-sha1 = "5524e27323cf4c4505699c3fb008c3f772269945"
 uuid = "8e7c35d0-a365-5155-bbbb-fb81a777f24e"
-version = "0.16.8"
+version = "0.16.9"
 
 [[ChainRulesCore]]
 deps = ["Compat", "LinearAlgebra", "SparseArrays"]
-git-tree-sha1 = "a325370b9dd0e6bf5656a6f1a7ae80755f8ccc46"
+git-tree-sha1 = "d9e40e3e370ee56c5b57e0db651d8f92bce98fea"
 uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
-version = "1.7.2"
+version = "1.10.1"
 
 [[CodecZlib]]
 deps = ["TranscodingStreams", "Zlib_jll"]
@@ -130,9 +130,9 @@ version = "1.11.1"
 
 [[FillArrays]]
 deps = ["LinearAlgebra", "Random", "SparseArrays", "Statistics"]
-git-tree-sha1 = "29890dfbc427afa59598b8cfcc10034719bd7744"
+git-tree-sha1 = "8756f9935b7ccc9064c6eef0bff0ad643df733a3"
 uuid = "1a297f60-69ca-5386-bcde-b61e274b549b"
-version = "0.12.6"
+version = "0.12.7"
 
 [[FiniteDiff]]
 deps = ["ArrayInterface", "LinearAlgebra", "Requires", "SparseArrays", "StaticArrays"]
@@ -141,16 +141,16 @@ uuid = "6a86dc24-6348-571c-b903-95158fe2bd41"
 version = "2.8.1"
 
 [[ForwardDiff]]
-deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "NaNMath", "Printf", "Random", "SpecialFunctions", "StaticArrays"]
-git-tree-sha1 = "c4203b60d37059462af370c4f3108fb5d155ff13"
+deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "NaNMath", "Preferences", "Printf", "Random", "SpecialFunctions", "StaticArrays"]
+git-tree-sha1 = "63777916efbcb0ab6173d09a658fb7f2783de485"
 uuid = "f6369f11-7733-5829-9624-2563aa707210"
-version = "0.10.20"
+version = "0.10.21"
 
 [[Gridap]]
 deps = ["AbstractTrees", "BSON", "BlockArrays", "Combinatorics", "DocStringExtensions", "FastGaussQuadrature", "FileIO", "FillArrays", "ForwardDiff", "JLD2", "JSON", "LineSearches", "LinearAlgebra", "NLsolve", "NearestNeighbors", "QuadGK", "Random", "SparseArrays", "SparseMatricesCSR", "StaticArrays", "Test", "WriteVTK"]
 git-tree-sha1 = "d3bfeb769259b08f7385bdad91681f38c50fc519"
 repo-rev = "gridap_distributed"
-repo-url = "https://github.com/gridap/Gridap.jl.git"
+repo-url = "https://github.com/gridap/Gridap.jl"
 uuid = "56d4f2e9-7ea1-5844-9cf6-b9c51ca7ce8e"
 version = "0.17.0"
 
@@ -163,10 +163,16 @@ version = "0.1.0"
 deps = ["Markdown"]
 uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 
+[[InverseFunctions]]
+deps = ["Test"]
+git-tree-sha1 = "f0c6489b12d28fb4c2103073ec7452f3423bd308"
+uuid = "3587e190-3f89-42d0-90ee-14403ec27112"
+version = "0.1.1"
+
 [[IrrationalConstants]]
-git-tree-sha1 = "f76424439413893a832026ca355fe273e93bce94"
+git-tree-sha1 = "7fd44fd4ff43fc60815f8e764c0f352b83c49151"
 uuid = "92d709cd-6900-40b7-9082-c6be49f344b6"
-version = "0.1.0"
+version = "0.1.1"
 
 [[IterativeSolvers]]
 deps = ["LinearAlgebra", "Printf", "Random", "RecipesBase", "SparseArrays"]
@@ -176,9 +182,9 @@ version = "0.9.1"
 
 [[JLD2]]
 deps = ["DataStructures", "FileIO", "MacroTools", "Mmap", "Pkg", "Printf", "Reexport", "TranscodingStreams", "UUIDs"]
-git-tree-sha1 = "192934b3e2a94e897ce177423fd6cf7bdf464bce"
+git-tree-sha1 = "46b7834ec8165c541b0b5d1c8ba63ec940723ffb"
 uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
-version = "0.4.14"
+version = "0.4.15"
 
 [[JLLWrappers]]
 deps = ["Preferences"]
@@ -234,10 +240,10 @@ deps = ["Libdl"]
 uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 
 [[LogExpFunctions]]
-deps = ["ChainRulesCore", "DocStringExtensions", "IrrationalConstants", "LinearAlgebra"]
-git-tree-sha1 = "34dc30f868e368f8a17b728a1238f3fcda43931a"
+deps = ["ChainRulesCore", "DocStringExtensions", "InverseFunctions", "IrrationalConstants", "LinearAlgebra"]
+git-tree-sha1 = "6193c3815f13ba1b78a51ce391db8be016ae9214"
 uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688"
-version = "0.3.3"
+version = "0.3.4"
 
 [[Logging]]
 uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
@@ -335,13 +341,13 @@ version = "0.12.3"
 
 [[Parsers]]
 deps = ["Dates"]
-git-tree-sha1 = "a8709b968a1ea6abc2dc1967cb1db6ac9a00dfb6"
+git-tree-sha1 = "98f59ff3639b3d9485a03a72f3ab35bab9465720"
 uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
-version = "2.0.5"
+version = "2.0.6"
 
 [[PartitionedArrays]]
 deps = ["Distances", "IterativeSolvers", "LinearAlgebra", "MPI", "Printf", "SparseArrays", "SparseMatricesCSR"]
-git-tree-sha1 = "763146e768b343fec44c521f5c7822116adc24ad"
+git-tree-sha1 = "4b79fc86fd30f5063d97d9c40291976ede6fd4d5"
 repo-rev = "gridap_distributed"
 repo-url = "https://github.com/fverdugo/PartitionedArrays.jl.git"
 uuid = "5a9dfac6-5c52-46f7-8278-5e2210713be9"
@@ -409,16 +415,16 @@ deps = ["LinearAlgebra", "Random"]
 uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 
 [[SparseMatricesCSR]]
-deps = ["LinearAlgebra", "SparseArrays"]
-git-tree-sha1 = "dd6dda1484d4031a47d27614ceeb89f511cd9ca4"
+deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"]
+git-tree-sha1 = "ad906b39ce5e05ec509495dfc7b38d11ce9ab40b"
 uuid = "a0a7dd2c-ebf4-11e9-1f05-cf50bc540ca1"
-version = "0.6.1"
+version = "0.6.5"
 
 [[SpecialFunctions]]
 deps = ["ChainRulesCore", "IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"]
-git-tree-sha1 = "793793f1df98e3d7d554b65a107e9c9a6399a6ed"
+git-tree-sha1 = "2d57e14cd614083f132b6224874296287bfa3979"
 uuid = "276daf66-3868-5448-9aa4-cd146d93841b"
-version = "1.7.0"
+version = "1.8.0"
 
 [[Static]]
 deps = ["IfElse"]
@@ -441,6 +447,10 @@ git-tree-sha1 = "1958272568dc176a1d881acb797beb909c785510"
 uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0"
 version = "1.0.0"
 
+[[SuiteSparse]]
+deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"]
+uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9"
+
 [[TOML]]
 deps = ["Dates"]
 uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"

Project.toml

@@ -13,7 +13,9 @@ SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 WriteVTK = "64499a7a-5c06-52f2-abe2-ccb03c286192"
 
 [compat]
+FillArrays = "0.8.4, 0.9, 0.10, 0.11, 0.12"
 Gridap = "0.17"
+MPI = "0.16, 0.17, 0.18, 0.19"
 PartitionedArrays = "0.2"
 julia = "1.3"
 

src/Algebra.jl

@@ -380,37 +380,47 @@ function local_views(a::PVectorCounter,rows)
   a.counters
 end
 
-function Arrays.nz_allocation(a::PVectorCounter)
+function Arrays.nz_allocation(a::PVectorCounter{<:FullyAssembledRows})
   dofs = a.rows
   values = map_parts(nz_allocation,a.counters)
-  PVectorAllocation(a.par_strategy,values,dofs)
+  PVectorAllocationTrackOnlyValues(a.par_strategy,values,dofs)
 end
 
-struct PVectorAllocation{A,B,C}
+struct PVectorAllocationTrackOnlyValues{A,B,C}
   par_strategy::A
   values::B
   rows::C
 end
 
-function local_views(a::PVectorAllocation)
+function local_views(a::PVectorAllocationTrackOnlyValues)
   a.values
 end
 
-function local_views(a::PVectorAllocation,rows)
+function local_views(a::PVectorAllocationTrackOnlyValues,rows)
   @check rows === a.rows
   a.values
 end
 
-function Algebra.create_from_nz(a::PVectorAllocation{<:FullyAssembledRows})
-  @notimplemented
+function Algebra.create_from_nz(a::PVectorAllocationTrackOnlyValues{<:FullyAssembledRows})
+  # Create PRange for the rows of the linear system
+  parts = get_part_ids(a.values)
+  ngdofs = length(a.rows)
+  nodofs = map_parts(num_oids,a.rows.partition)
+  first_grdof = map_parts(first_gdof_from_ids,a.rows.partition)
+
+  # This one has no ghost rows
+  rows = PRange(parts,ngdofs,nodofs,first_grdof)
+
+  _rhs_callback(a,rows)
 end
 
-function Algebra.create_from_nz(a::PVectorAllocation{<:SubAssembledRows})
-  @notimplemented
+function Algebra.create_from_nz(a::PVectorAllocationTrackOnlyValues{<:SubAssembledRows})
+  # This point MUST NEVER be reached. If reached there is an inconsistency
+  # in the parallel code in charge of vector assembly
+  @assert false
 end
 
 function _rhs_callback(c_fespace,rows)
-
   # The ghost values in b_fespace are aligned with the FESpace
   # but not with the ghost values in the rows of A
   b_fespace = PVector(c_fespace.values,c_fespace.rows)
@@ -442,14 +452,14 @@ end
 
 function Algebra.create_from_nz(a::PVector)
   # For FullyAssembledRows the underlying Exchanger should
-  # not have ghost layer making asseble! do nothing (TODO check)
+  # not have ghost layer making assemble! do nothing (TODO check)
   assemble!(a)
   a
 end
 
 function Algebra.create_from_nz(
   a::DistributedAllocationCOO{<:FullyAssembledRows},
-  c_fespace::PVectorAllocation{<:FullyAssembledRows})
+  c_fespace::PVectorAllocationTrackOnlyValues{<:FullyAssembledRows})
 
   function callback(rows)
     _rhs_callback(c_fespace,rows)
@@ -459,9 +469,15 @@ function Algebra.create_from_nz(
   A,b
 end
 
+struct PVectorAllocationTrackTouchedAndValues{A,B,C}
+  allocations::A
+  values::B
+  rows::C
+end
+
 function Algebra.create_from_nz(
   a::DistributedAllocationCOO{<:SubAssembledRows},
-  c_fespace::PVectorAllocation{<:SubAssembledRows})
+  c_fespace::PVectorAllocationTrackOnlyValues{<:SubAssembledRows})
 
   function callback(rows)
     _rhs_callback(c_fespace,rows)
@@ -476,3 +492,119 @@ function Algebra.create_from_nz(
   A,b
 end
 
+struct ArrayAllocationTrackTouchedAndValues{A}
+  touched::Vector{Bool}
+  values::A
+end
+
+Gridap.Algebra.LoopStyle(::Type{<:ArrayAllocationTrackTouchedAndValues}) = Gridap.Algebra.Loop()
+
+
+function local_views(a::PVectorAllocationTrackTouchedAndValues,rows)
+  @check rows === a.rows
+  a.allocations
+end
+
+@inline function Arrays.add_entry!(c::Function,a::ArrayAllocationTrackTouchedAndValues,v,i,j)
+  @notimplemented
+end
+@inline function Arrays.add_entry!(c::Function,a::ArrayAllocationTrackTouchedAndValues,v,i)
+  if i>0
+    if !(a.touched[i])
+      a.touched[i]=true
+    end
+    if v!=nothing
+      a.values[i]=c(v,a.values[i])
+    end
+  end
+  nothing
+end
+@inline function Arrays.add_entries!(c::Function,a::ArrayAllocationTrackTouchedAndValues,v,i,j)
+  @notimplemented
+end
+@inline function Arrays.add_entries!(c::Function,a::ArrayAllocationTrackTouchedAndValues,v,i)
+  if v != nothing
+    for (ve,ie) in zip(v,i)
+      Arrays.add_entry!(c,a,ve,ie)
+    end
+  else
+    for ie in i
+      Arrays.add_entry!(c,a,nothing,ie)
+    end
+  end
+  nothing
+end
+
+function Arrays.nz_allocation(a::DistributedCounterCOO{<:SubAssembledRows},
+                              b::PVectorCounter{<:SubAssembledRows})
+  A = nz_allocation(a)
+  dofs = b.rows
+  values = map_parts(nz_allocation,b.counters)
+  B=PVectorAllocationTrackOnlyValues(b.par_strategy,values,dofs)
+  A,B
+end
+
+function Arrays.nz_allocation(a::PVectorCounter{<:SubAssembledRows})
+  dofs = a.rows
+  values = map_parts(nz_allocation,a.counters)
+  touched = map_parts(values) do values
+     fill!(Vector{Bool}(undef,length(values)),false)
+  end
+  allocations=map_parts(values,touched) do values,touched
+    ArrayAllocationTrackTouchedAndValues(touched,values)
+  end
+  PVectorAllocationTrackTouchedAndValues(allocations,values,dofs)
+end
+
+function local_views(a::PVectorAllocationTrackTouchedAndValues)
+  a.allocations
+end
+
+function Algebra.create_from_nz(a::PVectorAllocationTrackTouchedAndValues)
+   parts = get_part_ids(a.values)
+   rdofs = a.rows # dof ids of the test space
+   ngrdofs = length(rdofs)
+   nordofs = map_parts(num_oids,rdofs.partition)
+   first_grdof = map_parts(first_gdof_from_ids,rdofs.partition)
+   rneigs_snd = rdofs.exchanger.parts_snd
+   rneigs_rcv = rdofs.exchanger.parts_rcv
+
+   # Find the ghost rows
+   hrow_to_hrdof=map_parts(local_views(a.allocations),rdofs.partition) do allocation, indices
+    lids_touched=findall(allocation.touched)
+    nhlids = count((x)->indices.lid_to_ohid[x]<0,lids_touched)
+    hlids = Vector{Int32}(undef,nhlids)
+    cur=1
+    for lid in lids_touched
+      hlid=indices.lid_to_ohid[lid]
+      if hlid<0
+        hlids[cur]=-hlid
+        cur=cur+1
+      end
+    end
+    hlids
+   end
+   hrow_to_gid, hrow_to_part = map_parts(
+       find_gid_and_part,hrow_to_hrdof,rdofs.partition)
+
+   # Create the range for rows
+   rows = PRange(
+           parts,
+           ngrdofs,
+           nordofs,
+           first_grdof,
+           hrow_to_gid,
+           hrow_to_part,
+           rneigs_snd,
+           rneigs_rcv)
+
+   b = _rhs_callback(a,rows)
+   t2 = async_assemble!(b)
+
+   # Wait the transfer to finish
+   if t2 !== nothing
+     map_parts(schedule,t2)
+     map_parts(wait,t2)
+   end
+   b
+end