Skip to main content
  • Home
  • Development
  • Documentation
  • Donate
  • Operational login
  • Browse the archive

swh logo
SoftwareHeritage
Software
Heritage
Archive
Features
  • Search

  • Downloads

  • Save code now

  • Add forge now

  • Help

Revision d1bb46dfe8bc7efe6257b5ce222c04bfe8aedaab authored by Ryan Wick on 10 May 2018, 23:16:07 UTC, committed by Ryan Wick on 10 May 2018, 23:16:07 UTC
Fix typo
1 parent 260bb4d
  • Files
  • Changes
  • c0d2609
  • /
  • src
  • /
  • kmers.cpp
Raw File Download

To reference or cite the objects present in the Software Heritage archive, permalinks based on SoftWare Hash IDentifiers (SWHIDs) must be used.
Select below a type of object currently browsed in order to display its associated SWHID and permalink.

  • revision
  • directory
  • content
revision badge
swh:1:rev:d1bb46dfe8bc7efe6257b5ce222c04bfe8aedaab
directory badge
swh:1:dir:3a1cefd215cbb8c5c686cc840a95178f90953d5e
content badge
swh:1:cnt:abf1c4d898de0de881ba27f78af17e278a5dd7f6

This interface enables to generate software citations, provided that the root directory of browsed objects contains a citation.cff or codemeta.json file.
Select below a type of object currently browsed in order to generate citations for them.

  • revision
  • directory
  • content
(requires biblatex-software package)
Generating citation ...
(requires biblatex-software package)
Generating citation ...
(requires biblatex-software package)
Generating citation ...
kmers.cpp
// Copyright 2017 Ryan Wick

// This file is part of Filtlong

// Filtlong is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later
// version.

// Filtlong is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
// details.

// You should have received a copy of the GNU General Public License along with Filtlong.  If not, see
// <http://www.gnu.org/licenses/>.


#include "kmers.h"

#include <iostream>
#include <zlib.h>
#include <stdio.h>
#include "kseq.h"
#include "misc.h"

KSEQ_INIT(gzFile, gzread)


Kmers::Kmers() {
    bloom_parameters parameters;

    // TO DO: it might be worth experimenting with these values to see how it affects time and memory usage.
    parameters.projected_element_count = 100000000;
    parameters.false_positive_probability = 0.0001; // 1 in 10000
    parameters.random_seed = 0xA5A5A5A5;

    parameters.compute_optimal_parameters();

    //Instantiate Bloom Filter
    bloom = new bloom_filter(parameters);

    required_kmer_copies = 4;
}


Kmers::~Kmers() {
    delete bloom;
}


void Kmers::add_read_fastqs(std::vector<std::string> filenames) {
    std::cerr << "Hashing 16-mers from Illumina reads\n";

    int sequence_count = 0;
    for (auto & filename : filenames)
        sequence_count += add_reference(filename, true);
    std::cerr << "  " << int_to_string(sequence_count) << " reads, "
              << int_to_string(m_kmers.size()) << " 16-mers\n\n";
}


void Kmers::add_assembly_fasta(std::string filename) {
    std::cerr << "Hashing 16-mers from assembly\n";
    std::cerr << "  " << filename << "\n";
    int sequence_count = add_reference(filename, false);
    std::string noun;
    if (sequence_count == 1)
        noun = "contig";
    else
        noun = "contigs";
    std::cerr << "  " << int_to_string(sequence_count) << " " << noun << ", "
              << int_to_string(m_kmers.size()) << " 16-mers\n\n";
}


int Kmers::add_reference(std::string filename, bool require_two_kmer_copies) {
    int l;
    uint32_t forward_kmer, reverse_kmer;
    int sequence_count = 0;

    // We'll use a different k-mer adding function for assembly hashing and read hashing.
    void (Kmers::*add_kmer)(uint32_t);
    if (require_two_kmer_copies)
        add_kmer = &Kmers::add_kmer_require_multiple_copies;
    else
        add_kmer = &Kmers::add_kmer_require_one_copy;

    long long base_count = 0;
    long long last_progress = 0;

    gzFile fp = gzopen(filename.c_str(), "r");
    kseq_t * seq = kseq_init(fp);
    while ((l = kseq_read(seq)) >= 0) {
        if (l == -3)
            std::cerr << "Error reading " << filename << "\n";
        else {
            ++sequence_count;

            // Can't get a 16-mer from a sequence shorter than 16 bp.
            if (seq->seq.l < 16)
                continue;

            base_count += seq->seq.l;
            char * sequence = seq->seq.s;

            // Build the starting k-mers from the first 16 bases.
            forward_kmer = starting_kmer_to_bits_forward(sequence);
            reverse_kmer = starting_kmer_to_bits_reverse(sequence);

            (this->*add_kmer)(forward_kmer);
            (this->*add_kmer)(reverse_kmer);

            for (size_t i = 16; i < seq->seq.l; ++i) {
                forward_kmer <<= 2;
                forward_kmer |= base_to_bits_forward(sequence[i]);

                reverse_kmer >>= 2;
                reverse_kmer |= base_to_bits_reverse(sequence[i]);

                (this->*add_kmer)(forward_kmer);
                (this->*add_kmer)(reverse_kmer);
            }

            if (base_count - last_progress >= 483611) {  // a big prime number so progress updates don't round off
                last_progress = base_count;
                print_hash_progress(filename, base_count);
            }
        }
    }
    kseq_destroy(seq);
    gzclose(fp);
    print_hash_progress(filename, base_count);
    std::cerr << "\n";
    return sequence_count;
}


void Kmers::add_kmer_require_one_copy(uint32_t kmer) {
    m_kmers.insert(kmer);
}


void Kmers::add_kmer_require_multiple_copies(uint32_t kmer) {
    // If the kmer is already in the final set, then we can skip the rest of this function.
    if (m_kmers.find(kmer) != m_kmers.end())
        return;

    // Check the bloom filter. If it's not in there, this is definitely the first time it's been seen.
    if (!bloom->contains(kmer))
        bloom->insert(kmer);

    // If it's in the bloom filter, then it's probably been seen once before (though maybe not, based on the false
    // positive rate of the bloom filter. Next we check the k-mer counts. If it's not in there, we say it's the second
    // time the kmer's been seen.
    else if (m_kmer_counts.find(kmer) == m_kmer_counts.end())
        m_kmer_counts[kmer] = 2;

    // If the k-mer is in the counts, then we increment its count. If the count is high enough, we add it to the k-mer
    // set (and remove it from the counts to save some memory).
    else {
        int times_seen = ++m_kmer_counts[kmer];
        if (times_seen >= required_kmer_copies) {
            m_kmers.insert(kmer);
            m_kmer_counts.erase(kmer);
        }
    }
}



bool Kmers::is_kmer_present(uint32_t kmer) {
    return m_kmers.find(kmer) != m_kmers.end();
}



uint32_t Kmers::base_to_bits_forward(char base) {
    switch (base) {
        case 'A':
            return 0;  // 00000000000000000000000000000000
        case 'C':
            return 1;  // 00000000000000000000000000000001
        case 'G':
            return 2;  // 00000000000000000000000000000010
        case 'T':
            return 3;  // 00000000000000000000000000000011
        case 'a':
            return 0;
        case 'c':
            return 1;
        case 'g':
            return 2;
        case 't':
            return 3;
    }
    return 0;
}


uint32_t Kmers::base_to_bits_reverse(char base) {
    switch (base) {
        case 'T':
            return 0;           // 00000000000000000000000000000000
        case 'G':
            return 1073741824;  // 01000000000000000000000000000000
        case 'C':
            return 2147483648;  // 10000000000000000000000000000000
        case 'A':
            return 3221225472;  // 11000000000000000000000000000000
        case 't':
            return 0;
        case 'g':
            return 1073741824;
        case 'c':
            return 2147483648;
        case 'a':
            return 3221225472;
    }
    return 0;
}


uint32_t Kmers::starting_kmer_to_bits_forward(char * sequence) {
    uint32_t kmer = 0;
    for (int i = 0; i < 16; ++i) {
        kmer <<= 2;
        kmer |= base_to_bits_forward(sequence[i]);
    }
    return kmer;
}


uint32_t Kmers::starting_kmer_to_bits_reverse(char * sequence) {
    uint32_t kmer = 0;
    for (int i = 0; i < 16; ++i) {
        kmer >>= 2;
        kmer |= base_to_bits_reverse(sequence[i]);
    }
    return kmer;
}
The diff you're trying to view is too large. Only the first 1000 changed files have been loaded.
Showing with 0 additions and 0 deletions (0 / 0 diffs computed)
swh spinner

Computing file changes ...

back to top

Software Heritage — Copyright (C) 2015–2026, The Software Heritage developers. License: GNU AGPLv3+.
The source code of Software Heritage itself is available on our development forge.
The source code files archived by Software Heritage are available under their own copyright and licenses.
Terms of use: Archive access, API— Content policy— Contact— JavaScript license information— Web API