Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Log too much mapping #316

Open
wants to merge 13 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/common/progress.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ class ProgressMeter {
if (logger.joinable()) {
logger.join();
}
uint64_t final_completed = completed.load();
completed.store(total);
do_print();
std::cerr << std::endl;
Expand Down
28 changes: 16 additions & 12 deletions src/map/include/computeMap.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ namespace skch
}

// Update progress after processing the fragment
fragment->output->progress.increment(fragment->len);
fragment->output->progress.increment(1);

fragment->fragments_processed->fetch_add(1, std::memory_order_relaxed);
delete fragment;
Expand Down Expand Up @@ -427,7 +427,6 @@ namespace skch
InputSeqProgContainer* input = nullptr;
if (input_queue.try_pop(input)) {
auto output = mapModule(input, fragment_queue);
//progress.increment(input->len / 4);
while (!merged_queue.try_push(output)) {
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
Expand Down Expand Up @@ -535,11 +534,17 @@ namespace skch
this->querySequenceNames = idManager->getQuerySequenceNames();
this->targetSequenceNames = idManager->getTargetSequenceNames();

// Count the total number of sequences and sequence length
uint64_t total_seqs = querySequenceNames.size();
uint64_t total_seq_length = 0;
// Count total fragments across all queries
uint64_t total_fragments = 0;
for (const auto& seqName : querySequenceNames) {
total_seq_length += idManager->getSequenceLength(idManager->getSequenceId(seqName));
auto len = idManager->getSequenceLength(idManager->getSequenceId(seqName));
// Count full fragments
int noOverlapFragmentCount = len / param.segLength;
// Add final fragment if needed
if (noOverlapFragmentCount >= 1 && len % param.segLength != 0) {
++noOverlapFragmentCount;
}
total_fragments += noOverlapFragmentCount;
}

std::vector<std::vector<std::string>> target_subsets = createTargetSubsets(targetSequenceNames);
Expand Down Expand Up @@ -609,7 +614,7 @@ namespace skch
std::atomic<bool> reader_done(false);
std::atomic<bool> workers_done(false);
std::atomic<bool> fragments_done(false);
processSubset(subset_count, target_subsets.size(), total_seq_length, input_queue, merged_queue,
processSubset(subset_count, target_subsets.size(), total_fragments, input_queue, merged_queue,
fragment_queue, reader_done, workers_done, fragments_done, combinedMappings);
}

Expand Down Expand Up @@ -682,14 +687,14 @@ namespace skch

}

void processSubset(uint64_t subset_count, size_t total_subsets, uint64_t total_seq_length,
void processSubset(uint64_t subset_count, size_t total_subsets, uint64_t total_fragments,
input_atomic_queue_t& input_queue, merged_mappings_queue_t& merged_queue,
fragment_atomic_queue_t& fragment_queue, std::atomic<bool>& reader_done,
std::atomic<bool>& workers_done, std::atomic<bool>& fragments_done,
std::unordered_map<seqno_t, MappingResultsVector_t>& combinedMappings)
{
progress_meter::ProgressMeter progress(
total_seq_length,
total_fragments * 2,
"[wfmash::mashmap] mapping ("
+ std::to_string(subset_count + 1) + "/" + std::to_string(total_subsets) + ")");

Expand Down Expand Up @@ -940,6 +945,7 @@ namespace skch
fragments.push_back(fragment);
}


if (noOverlapFragmentCount >= 1 && input->len % param.segLength != 0) {
auto fragment = new FragmentData{
&(input->seq)[0u] + input->len - param.segLength,
Expand All @@ -964,7 +970,7 @@ namespace skch
}

// Wait for all fragments to be processed
while (fragments_processed.load(std::memory_order_relaxed) < noOverlapFragmentCount) {
while (fragments_processed.load(std::memory_order_seq_cst) < noOverlapFragmentCount) {
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}

Expand Down Expand Up @@ -1904,7 +1910,6 @@ namespace skch
}

fragment.blockLength = std::max(fragment.refEndPos - fragment.refStartPos, fragment.queryEndPos - fragment.queryStartPos);
//fragment.blockLength = fragment.queryEndPos - fragment.queryStartPos;

fragment.approxMatches = std::round(fragment.nucIdentity * fragment.blockLength / 100.0);

Expand Down Expand Up @@ -2074,7 +2079,6 @@ namespace skch
best_it2->chainPairScore = best_score;
best_it2->chainPairId = it->splitMappingId;
}
progress.increment(1);
}

// Assign the merged mapping ids
Expand Down