Skip to content

Developload estimation #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
danflomin wants to merge 2 commits into
base: develop-test1
Choose a base branch
from
Open

Developload estimation #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
57886d0
this version uses 2 files for ranks and binning. Now added files for …
Apr 7, 2021
3511b05
add code. during inspection of ratio and distinct kmers methods
Apr 18, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions include/gerbil/Bundle.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

#include "KMer.h"


namespace gerbil {


Expand Down Expand Up @@ -107,6 +108,8 @@ namespace gerbil {
uint64 sMerNumber; // number of s-mers
uint64 kMerNumber; // number of k-mers



SuperBundle();

~SuperBundle();
Expand All @@ -123,6 +126,8 @@ namespace gerbil {

bool merge(const SuperBundle &sb);



//debugging, working only with "add"
uint32 getSize() {
return _next - data + 1;
Expand Down
7 changes: 4 additions & 3 deletions include/gerbil/CpuHasher.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -453,16 +453,17 @@ namespace gerbil {
// preparation next bin
if (notEmpty) {
binKMers = 0;
std::cout << binUKMers << " load in " << curTempFileId << std::endl;
binUKMers = 0;
binFKMers = 0;
ratio = distributor->getSplitRatio(false, tId, curTempFileId);
//printf("cpu hasher thread %i: my ratio is %f\n", tId, ratio);
curTempFileId = kmb->getTempFileId();
curTempRun = kmb->getTempFileRun();
apprUKMersNumber = _tempFiles[curTempFileId].approximateUniqueKmers(
kMersNumber ? (double) uKMersNumber / kMersNumber : START_RATIO) * ratio / _tempFiles[curTempFileId].getNumberOfRuns();
apprUKMersNumber = _tempFiles[curTempFileId].approximateUniqueKmers(kMersNumber ? (double) uKMersNumber / kMersNumber : START_RATIO) * ratio / _tempFiles[curTempFileId].getNumberOfRuns(); // lolz put here approximation of kmers
//std::cout << "approx load" << apprUKMersNumber << " id " << curTempFileId << std::endl;
//std::cout << "curTempFileId : " << curTempFileId << " curTempRun: " << curTempRun << std::endl;
KMCHT_setPartSize();
KMCHT_setPartSize(); // lolz
// reset timer
duration = ms(0);
}
Expand Down
7 changes: 4 additions & 3 deletions include/gerbil/KmerHasher.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ namespace gerbil {

if (_norm) {
// use normalized kmers
processThreadSplit<K, true>(tId, cpuKMerQueues, gpuKMerQueues);
processThreadSplit<K, true>(tId, cpuKMerQueues, gpuKMerQueues); /// lolz push here stats
} else {
// do not use normalized kmers
processThreadSplit<K, false>(tId, cpuKMerQueues, gpuKMerQueues);
Expand All @@ -130,7 +130,7 @@ namespace gerbil {
cpu::HasherTask<K> cpuHasher(_numCPUHasher,
_distributor,
&_kmcSyncSwapQueue, _tempFiles, _thresholdMin,
_maxKmcHashtableSize, _tempFolder);
_maxKmcHashtableSize, _tempFolder); /// lolz???

// create Hash Tables (GPU-side)
gpu::HasherTask<K> gpuHasher(_numGPUHasher,
Expand Down Expand Up @@ -159,7 +159,7 @@ namespace gerbil {
}

// start splitting
processSplit<K>(cpuKMerQueues, gpuKMerQueues);
processSplit<K>(cpuKMerQueues, gpuKMerQueues); /// lolz push here stats

// start hashing
cpuHasher.hash(cpuKMerQueues);
Expand Down Expand Up @@ -203,6 +203,7 @@ namespace gerbil {
_btUKMersNumberCPU = cpuHasher.getBtUKMersNumber();
_kMersNumberGPU = gpuHasher.getKMersNumber();
_uKMersNumberGPU = gpuHasher.getUKMersNumber();
std::cout << _uKMersNumberGPU << std::endl;
_btUKMersNumberGPU = gpuHasher.getBtUKMersNumber();
});

Expand Down
6 changes: 6 additions & 0 deletions include/gerbil/TempFile.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#define BINFILE_H_

#include "Bundle.h"
#include "hyperloglog.h"

namespace gerbil {

Expand All @@ -42,6 +43,8 @@ namespace gerbil {

uint64 _numberOfRuns; // number of runs

HyperLogLog *hll; // HyperLogLog data structure for load estimation

public:

TempFile();
Expand Down Expand Up @@ -90,6 +93,9 @@ namespace gerbil {

inline uint64 TempFile::approximateUniqueKmers(const double ratio) const {
return _kmers * ratio;
// hll = new HyperLogLog(HLL_SIZE);
//return cardinality = (uint64)hll.estimate() + 1;
// delete hll;
}

inline const uint64 &TempFile::getKMersNumber() const {
Expand Down
2 changes: 1 addition & 1 deletion include/gerbil/config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ namespace gerbil {

#define DEF_TEMPFILES_NUMBER 512
#define MIN_TEMPFILES_NUMBER 2
#define MAX_TEMPFILES_NUMBER 4 * 1024
#define MAX_TEMPFILES_NUMBER 4 * 1024 * 100

#define DEF_THREADS_NUMBER 8
#define MIN_THREADS_NUMBER 4
Expand Down
Loading