clean up

Author: philippwindischhofer

src/DistributedNDArray.hxx

@@ -87,10 +87,6 @@ void DistributedNDArray<T, dims, DenseT, SparseT, SerializerT>::WriteChunk(const
   char uuid_string[36];
   uuid_unparse(uuid_binary, uuid_string);
   std::string chunk_filename = std::string(uuid_string) + ".bin";
-
-  // if(chunk_filename.find_first_not_of("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890_-.") != std::string::npos) {    
-  //   throw std::runtime_error("Error: non-compliant chunk filename detected!");
-  // }
 
   IndexVector stop_ind = start_ind + chunk.shape();
 
@@ -111,22 +107,17 @@ void DistributedNDArray<T, dims, DenseT, SparseT, SerializerT>::WriteChunk(const
   // std::size_t sparse_vs_dense_expense_ratio = 3; // when only counting storage space: sparse storage is approximately 3x as expensive as dense storage per nonzero element
   std::size_t sparse_vs_dense_expense_ratio = 20; // when also counting complexity of deserializing + rebuilding a dense chunk
   if(sparse_vs_dense_expense_ratio * num_nonzero_elems < num_elems) {
-    
-    // std::cout << "going to sparsify" << std::endl;
-    
+        
     auto to_keep = [](scalar_t value) -> bool {
       return value != 0.0;
     };        
     sparse_t sparse_chunk = sparse_t::From(chunk, to_keep, 0.0);    
     meta.chunk_type = ChunkType::sparse;
-
-    // std::cout << "after sparsification, " << sparse_chunk.NumEntries() << " entries remain" << std::endl;
 
     m_ser.template serialize<ChunkMetadata>(ofs, meta);   
     m_ser.template serialize<sparse_t>(ofs, sparse_chunk);
   }
-  else {  
-    // std::cout << "store as dense" << std::endl;
+  else {
     m_ser.template serialize<ChunkMetadata>(ofs, meta);   
     m_ser.template serialize<dense_t>(ofs, chunk);
   }
@@ -168,65 +159,30 @@ void DistributedNDArray<T, dims, DenseT, SparseT, SerializerT>::rebuildIndex() {
     ifs.open(dir_entry.path(), std::ios::in | std::ios::binary);
     ChunkMetadata meta = m_ser.template deserialize<ChunkMetadata>(ifs);
     ifs.close();
-
-    // std::cout << "found metadata containing filename: '" + meta.filename + "' from path '" + std::string(dir_entry.path()) + "'!" << std::endl;
-    
-    // if(meta.filename.find_first_not_of("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890_-.") != std::string::npos) {
-    //   throw std::runtime_error("Error: found metadata containing non-compliant filename: '" + meta.filename + "' from path '" + std::string(dir_entry.path()) + "'!");
-    // }
 
     m_chunk_index.push_back(meta);
   }
 }
 
 template <class T, std::size_t dims, template<class, std::size_t> class DenseT, template<class, std::size_t> class SparseT, class SerializerT>
 void DistributedNDArray<T, dims, DenseT, SparseT, SerializerT>::RebuildChunks(const IndexVector& requested_chunk_size) {
-
-  // How to make sure own index doesn't get messed up when new chunks are generated?
-
-  // 0) factor out most things of currently existing RegisterChunk into new private method:
-  //      -> WriteChunk(chunk, start_ind, add_to_index)
-  //   (keep the overlap checking etc. in the original RegisterChunk)
-  
-  // Then, this function will look like
-  // 0) rebuild index (to make sure everything is taken into account)
-  // 1) then loop over chunks like in `WeightingFieldUtils`
-  //     -> get chunk that has the starting indices
-  //     -> if it conforms to the requested size, don't touch it and continue (will guarantee this function is idempotent)
-  //     -> if it doesn't, build a chunk of correct size (using NDArray::range)
-  // 2) write the individual chunks, but don't touch the index (using `WriteChunk`)
-  // 3) remove all chunks in the index (-> this will remove all the old ones), BUT not those that haven't been touched because they are already conforming to the correct size
-
-  // std::cout << "in RebuildChunks" << std::endl;
 
   if(requested_chunk_size.size() != dims) {
     throw std::runtime_error("Error: requested chunk size has wrong dimensionality!");
   }
-
-  // std::cout << "rebuilding index" << std::endl;
 
   // Make sure we start from a clean index
   rebuildIndex(); 
-
-  // std::cout << "rebuilt index" << std::endl;
 
   if(!isGloballyContiguous(getGlobalStartInd(), getGlobalStopInd())) {
     throw std::runtime_error("Error: refusing to rebuild chunks for a non-contiguous array!");
   }
 
   calculateShape();
 
-  IndexVector global_shape(this -> m_shape);
-
-  // std::cout << "global shape" << std::endl;
-  // global_shape.print();
-  
+  IndexVector global_shape(this -> m_shape);  
   IndexVector number_required_chunks = (global_shape + requested_chunk_size - 1) / requested_chunk_size;
 
-  // std::cout << "will have " << std::endl;
-  // number_required_chunks.print();
-  // std::cout << " chunks after rebuilding" << std::endl;
-
   index_t chunks_to_keep;
 
   IndexVector buf_inds_start(dims, 0);
@@ -244,15 +200,11 @@ void DistributedNDArray<T, dims, DenseT, SparseT, SerializerT>::RebuildChunks(co
     dense_t current_chunk = retrieveChunk(chunk_index);
 
     if(actual_chunk_shape == current_chunk.shape()) {
-      // std::cout << "chunk already has the correct size, keep it" << std::endl;
       chunks_to_keep.push_back(m_chunk_index[chunk_index]);
       continue;
     }
 
-    // std::cout << "now working on rebuild chunk with inds" << std::endl;
-    // std::cout << "chunk_inds_start = " << std::endl;
     chunk_inds_start.print();
-    // std::cout << "chunk_inds_end = " << std::endl;
     chunk_inds_end.print();
 
     dense_t chunk = range(chunk_inds_start, chunk_inds_end);
@@ -278,30 +230,13 @@ void DistributedNDArray<T, dims, DenseT, SparseT, SerializerT>::RebuildChunks(co
 template <class T, std::size_t dims, template<class, std::size_t> class DenseT, template<class, std::size_t> class SparseT, class SerializerT>
 void DistributedNDArray<T, dims, DenseT, SparseT, SerializerT>::MergeChunks(std::size_t dim_to_merge, std::size_t max_dimsize) {
 
-  // 0) Rebuild index (to make sure we have the full picture)
-
-  // 2) Then, start with chunk that has the global start inds
-
-  // --> operate entirely on the index and prepare a full list of chunk mergings that need to happen (without actually doing anything)
-  //     --> remove any elements from the index that are already taken care of
-  // --> check if there are any problems; if not, go ahead and implement the chunk mergings
-  // --> go to the chunk that neighbours the current main one and continue
-  
-  // 3) again, have the to_keep mechanism
-  
-  // std::cout << "in MergeNeighbouringChunks" << std::endl;
-
   rebuildIndex(); 
-
-  // std::cout << "rebuilt index" << std::endl;
 
   if(!isGloballyContiguous(getGlobalStartInd(), getGlobalStopInd())) {
     throw std::runtime_error("Error: refusing to merge chunks for a non-contiguous array!");
   }
 
   calculateShape();
-
-  // std::cout << "have a total of " << m_chunk_index.size() << " chunks before merging" << std::endl;
 
   // put chunks in order along the merging axis
   std::vector<std::size_t> chunk_order(m_chunk_index.size());
@@ -338,14 +273,10 @@ void DistributedNDArray<T, dims, DenseT, SparseT, SerializerT>::MergeChunks(std:
     chunks_to_merge.push_back(retrieveChunk(cur_chunk_index));
     std::size_t output_chunk_shape = chunks_to_merge.back().shape(dim_to_merge);
     std::size_t neighbour_chunk_index = cur_chunk_index;
-
-    // std::cout << "start merging tracer from chunk with index = " << neighbour_chunk_index << std::endl;
 
     while(true) {
       try {
 	neighbour_chunk_index = getNeighbouringChunkIndex(neighbour_chunk_index, dim_to_merge);
-
-	// std::cout << "have neighbouring chunk with index = " << neighbour_chunk_index <<  " / " << m_chunk_index.size() << std::endl;
 
 	chunks_to_merge.push_back(retrieveChunk(neighbour_chunk_index));
 	std::erase(chunk_order, neighbour_chunk_index);
@@ -364,31 +295,7 @@ void DistributedNDArray<T, dims, DenseT, SparseT, SerializerT>::MergeChunks(std:
 
     // perform the merge and write the result to disk
     dense_t output_chunk = NDArrayOps::concatenate(chunks_to_merge, dim_to_merge);
-    WriteChunk(output_chunk, cur_chunk_meta.start_ind, false);
-    
-    // dense_t output_chunk = retrieveChunk(cur_chunk_index);
-    // std::size_t neighbour_chunk_index = cur_chunk_index;
-    // while(true) {
-    //   try {
-    // 	neighbour_chunk_index = getNeighbouringChunkIndex(neighbour_chunk_index, dim_to_merge);	
-    // 	std::erase(chunk_order, neighbour_chunk_index);
-
-    // 	std::cout << "merging chunk " << cur_chunk_index << " with chunk " << neighbour_chunk_index << std::endl;
-	  
-    // 	output_chunk = NDArrayOps::concatenate(output_chunk, retrieveChunk(neighbour_chunk_index), dim_to_merge);
-
-    // 	// chunk is now large enough
-    // 	if(output_chunk.shape(dim_to_merge) >= max_dimsize) {
-    // 	  break;
-    // 	}	
-    //   } catch(const ChunkNotFoundError& e) {
-    // 	// there are no more neighbours in this direction; stop here
-    // 	break;
-    //   }
-    // }
-
-    // // write it to disk
-    // WriteChunk(output_chunk, cur_chunk_meta.start_ind, false);
+    WriteChunk(output_chunk, cur_chunk_meta.start_ind, false);    
   }
 
   // remove all the chunks in the old index that should not be kept around
@@ -438,12 +345,6 @@ std::size_t DistributedNDArray<T, dims, DenseT, SparseT, SerializerT>::getNeighb
 
   std::size_t neighbour_chunk_ind = getChunkIndex(neighbour_chunk_start_ind);
 
-  // if(neighbour_chunk_ind > m_chunk_index.size()) {
-  //   std::cout << "MMMM Found out-of-bounds neighbouring chunk: " + std::to_string(neighbour_chunk_ind) + "/" + std::to_string(m_chunk_index.size()) << std::endl;
-  //   std::cout << "MMMM From request with neighbour_chunk_start_ind = " << std::endl;
-  //   neighbour_chunk_start_ind.print();
-  // }
-
   return neighbour_chunk_ind;
 }
 
@@ -502,18 +403,15 @@ std::size_t DistributedNDArray<T, dims, DenseT, SparseT, SerializerT>::getChunkI
 
   if(chunkContainsInds(m_chunk_index[m_chunk_last_accessed], inds)) {
     [[likely]];
-    // std::cout << "NNN m_chunk_last_accessed gave " << m_chunk_last_accessed << std::endl;
     return m_chunk_last_accessed;
   }
   else {
     // Trigger a full chunk lookup
     // TODO: have a search tree here with logarithmic instead of linear complexity
-    // std::cout << "NNN trigger full chunk lookup" << std::endl;
     std::size_t chunk_ind = 0;
     for(chunk_ind = 0; chunk_ind < m_chunk_index.size(); chunk_ind++) {
       if(chunkContainsInds(m_chunk_index[chunk_ind], inds)) {
 	m_chunk_last_accessed = chunk_ind;	
-	// std::cout << "NNN full chunk lookup gave " << chunk_ind << std::endl;
 	return chunk_ind;
       }
     }   
@@ -549,10 +447,6 @@ DistributedNDArray<T, dims, DenseT, SparseT, SerializerT>::dense_t& DistributedN
     std::fstream ifs;
     std::string chunk_path = m_dirpath + "/" + chunk_meta.filename;
     std::cout << "Loading chunk from " + chunk_path + " ... ";
-
-    // if(chunk_meta.filename.find_first_not_of("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890_-.") != std::string::npos) {    
-    //   throw std::runtime_error("Error: trying to open chunk " + std::to_string(chunk_ind) + "/" + std::to_string(m_chunk_index.size()) + " from file with non-compliant name: '" + chunk_meta.filename + "'!");
-    // }
 
     if(!std::filesystem::exists(chunk_path)) {
       throw std::runtime_error("Error: trying to access non-existant file '" + chunk_path + "'!");
@@ -566,7 +460,6 @@ DistributedNDArray<T, dims, DenseT, SparseT, SerializerT>::dense_t& DistributedN
     }
     else if(meta.chunk_type == ChunkType::sparse) {
       m_chunk_cache.insert({chunk_ind, dense_t::FromSparseFile(ifs)});
-      // m_chunk_cache.insert({chunk_ind, dense_t::From(m_ser.template deserialize<sparse_t>(ifs))});
     }
     else {
       throw std::runtime_error("Error: unknown chunk type encountered!");