From 2cf26ce3523c88ce1b16bb61f7fd59b6466f8023 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Wed, 29 Jul 2026 08:14:01 +0000 Subject: [PATCH 01/27] Add shared-filesystem parallel linclust foundations. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Björn Buschkämper --- src/CommandDeclarations.h | 1 + src/MMseqsBase.cpp | 12 + src/commons/CMakeLists.txt | 6 + src/commons/DenseIndex.cpp | 302 +++++++++++++ src/commons/DenseIndex.h | 108 +++++ src/commons/LengthRankedPlan.cpp | 207 +++++++++ src/commons/LengthRankedPlan.h | 116 +++++ src/commons/ParallelCoordination.cpp | 393 +++++++++++++++++ src/commons/ParallelCoordination.h | 178 ++++++++ src/commons/Parameters.cpp | 12 + src/commons/Parameters.h | 3 + src/linclust/CMakeLists.txt | 1 + src/linclust/KmerPartition.cpp | 174 ++++++++ src/linclust/KmerPartition.h | 176 ++++++++ src/test/CMakeLists.txt | 4 + src/test/TestDenseIndex.cpp | 176 ++++++++ src/test/TestKmerPartition.cpp | 281 ++++++++++++ src/test/TestLengthRankedPlan.cpp | 281 ++++++++++++ src/test/TestParallelCoordination.cpp | 299 +++++++++++++ src/util/CMakeLists.txt | 1 + src/util/createdbparallel.cpp | 592 ++++++++++++++++++++++++++ 21 files changed, 3323 insertions(+) create mode 100644 src/commons/DenseIndex.cpp create mode 100644 src/commons/DenseIndex.h create mode 100644 src/commons/LengthRankedPlan.cpp create mode 100644 src/commons/LengthRankedPlan.h create mode 100644 src/commons/ParallelCoordination.cpp create mode 100644 src/commons/ParallelCoordination.h create mode 100644 src/linclust/KmerPartition.cpp create mode 100644 src/linclust/KmerPartition.h create mode 100644 src/test/TestDenseIndex.cpp create mode 100644 src/test/TestKmerPartition.cpp create mode 100644 src/test/TestLengthRankedPlan.cpp create mode 100644 src/test/TestParallelCoordination.cpp create mode 100644 src/util/createdbparallel.cpp diff --git a/src/CommandDeclarations.h b/src/CommandDeclarations.h index 3a9bc7d0d..acc8b8042 100644 --- a/src/CommandDeclarations.h +++ b/src/CommandDeclarations.h @@ -26,6 +26,7 @@ extern int convertkb(int argc, const char **argv, const Command& command); extern int convertmsa(int argc, const char **argv, const Command& command); extern int convertprofiledb(int argc, const char **argv, const Command& command); extern int createdb(int argc, const char **argv, const Command& command); +extern int createdbparallel(int argc, const char **argv, const Command& command); extern int makepaddedseqdb(int argc, const char **argv, const Command& command); extern int createindex(int argc, const char **argv, const Command& command); extern int createlinindex(int argc, const char **argv, const Command& command); diff --git a/src/MMseqsBase.cpp b/src/MMseqsBase.cpp index cb8850ff5..1f8b8cf13 100644 --- a/src/MMseqsBase.cpp +++ b/src/MMseqsBase.cpp @@ -160,6 +160,18 @@ std::vector baseCommands = { " ... | ", CITATION_MMSEQS2, {{"fast[a|q]File[.gz|bz2]|stdin", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA | DbType::VARIADIC, &DbValidator::flatfileStdinAndGeneric }, {"sequenceDB", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::flatfile }}}, + {"createdbparallel", createdbparallel, &par.createdbparallel, COMMAND_DATABASE_CREATION, + "Convert FASTA file(s) to a length-ranked sequence DB with many workers", + "# Build a sequence DB with sequences ordered longest first and dense keys.\n" + "# Every worker runs this identical command line and coordinates through\n" + "# .coord; workers may join late, die and be restarted.\n" + "mmseqs createdbparallel seq.fasta sequenceDB\n\n" + "# Run it from several nodes against the same shared filesystem\n" + "srun -N 8 mmseqs createdbparallel seq.fasta sequenceDB --chunk-size 1G\n", + "Martin Steinegger ", + " ... ", + CITATION_MMSEQS2, {{"fastaFile", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA | DbType::VARIADIC, &DbValidator::flatfile }, + {"sequenceDB", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::flatfile }}}, {"makepaddedseqdb", makepaddedseqdb, &par.makepaddedseqdb, COMMAND_HIDDEN, "Generate a padded sequence DB", "Generate a padded sequence DB", diff --git a/src/commons/CMakeLists.txt b/src/commons/CMakeLists.txt index d013f56a2..52c726c3d 100644 --- a/src/commons/CMakeLists.txt +++ b/src/commons/CMakeLists.txt @@ -11,6 +11,8 @@ set(commons_header_files commons/DBConcat.h commons/DBReader.h commons/DBWriter.h + commons/DenseIndex.h + commons/LengthRankedPlan.h commons/IntervalArray.h commons/Debug.h commons/Domain.h @@ -34,6 +36,7 @@ set(commons_header_files commons/ProfileStates.h commons/LibraryReader.h commons/Parameters.h + commons/ParallelCoordination.h commons/PatternCompiler.h commons/ScoreMatrix.h commons/Sequence.h @@ -57,6 +60,8 @@ set(commons_source_files commons/DBConcat.cpp commons/DBReader.cpp commons/DBWriter.cpp + commons/DenseIndex.cpp + commons/LengthRankedPlan.cpp commons/Debug.cpp commons/ExpressionParser.cpp commons/FileUtil.cpp @@ -72,6 +77,7 @@ set(commons_source_files commons/NucleotideMatrix.cpp commons/Orf.cpp commons/Parameters.cpp + commons/ParallelCoordination.cpp commons/ProfileStates.cpp commons/LibraryReader.cpp commons/ScoreMatrix.cpp diff --git a/src/commons/DenseIndex.cpp b/src/commons/DenseIndex.cpp new file mode 100644 index 000000000..b6da0a53a --- /dev/null +++ b/src/commons/DenseIndex.cpp @@ -0,0 +1,302 @@ +#include "DenseIndex.h" + +#include "Debug.h" +#include "FileUtil.h" +#include "MemoryMapped.h" +#include "Util.h" + +#include +#include + +#include + +std::string DenseIndex::fileName(const std::string &dbName) { + return dbName + ".index.bin"; +} + +bool DenseIndex::exists(const std::string &dbName) { + return FileUtil::fileExists(fileName(dbName).c_str()); +} + +void DenseIndex::build(const std::string &dbName) { + const std::string textIndexName = dbName + ".index"; + MemoryMapped indexData(textIndexName, MemoryMapped::WholeFile, MemoryMapped::SequentialScan); + if (indexData.isValid() == false) { + Debug(Debug::ERROR) << "Cannot open index file " << textIndexName << "\n"; + EXIT(EXIT_FAILURE); + } + + const std::string outputName = fileName(dbName); + FILE *out = fopen(outputName.c_str(), "wb"); + if (out == NULL) { + Debug(Debug::ERROR) << "Cannot write dense index " << outputName << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + + // Reserve the header; it is rewritten at the end once the totals are known. + Header header; + memset(&header, 0, sizeof(header)); + if (fwrite(&header, sizeof(header), 1, out) != 1) { + Debug(Debug::ERROR) << "Cannot write dense index header " << outputName << "\n"; + EXIT(EXIT_FAILURE); + } + + char *cursor = (char *) indexData.getData(); + const size_t indexDataSize = indexData.size(); + size_t consumed = 0; + + const size_t bufferCapacity = 65536; + Entry *buffer = new Entry[bufferCapacity]; + size_t buffered = 0; + + uint64_t entryCount = 0; + uint64_t firstKey = 0; + uint64_t dataSize = 0; + uint32_t maxSeqLen = 0; + + const char *cols[3]; + while (consumed < indexDataSize) { + Util::getWordsOfLine(cursor, cols, 3); + const DBKeyType key = Util::fast_atoi(cols[0]); + const uint64_t offset = Util::fast_atoi(cols[1]); + const uint32_t length = Util::fast_atoi(cols[2]); + + if (entryCount == 0) { + firstKey = key; + } else if (key != firstKey + entryCount) { + // Row number and key must agree, otherwise a range load would return + // the wrong sequences without any way to notice. + Debug(Debug::ERROR) << "Database " << dbName << " does not have dense keys: expected key " + << (firstKey + entryCount) << " at row " << entryCount << ", found " + << key << ". A dense index requires keys numbered consecutively.\n"; + EXIT(EXIT_FAILURE); + } + + buffer[buffered].offset = offset; + buffer[buffered].length = length; + buffered++; + if (buffered == bufferCapacity) { + if (fwrite(buffer, sizeof(Entry), buffered, out) != buffered) { + Debug(Debug::ERROR) << "Cannot write dense index " << outputName << "\n"; + EXIT(EXIT_FAILURE); + } + buffered = 0; + } + + dataSize += length; + maxSeqLen = std::max(maxSeqLen, length); + entryCount++; + + cursor = Util::skipLine(cursor); + consumed = cursor - (char *) indexData.getData(); + } + + if (buffered > 0 && fwrite(buffer, sizeof(Entry), buffered, out) != buffered) { + Debug(Debug::ERROR) << "Cannot write dense index " << outputName << "\n"; + EXIT(EXIT_FAILURE); + } + delete[] buffer; + indexData.close(); + + header.magic = MAGIC; + header.version = VERSION; + header.entryCount = entryCount; + header.firstKey = firstKey; + header.dataSize = dataSize; + header.maxSeqLen = maxSeqLen; + if (fseek(out, 0, SEEK_SET) != 0 || fwrite(&header, sizeof(header), 1, out) != 1) { + Debug(Debug::ERROR) << "Cannot finalise dense index " << outputName << "\n"; + EXIT(EXIT_FAILURE); + } + if (fclose(out) != 0) { + Debug(Debug::ERROR) << "Cannot close dense index " << outputName << "\n"; + EXIT(EXIT_FAILURE); + } + + Debug(Debug::INFO) << "Dense index for " << dbName << ": " << entryCount << " entries, first key " + << firstKey << "\n"; +} + +DenseIndex::Info DenseIndex::readInfo(const std::string &dbName) { + const std::string path = fileName(dbName); + FILE *file = fopen(path.c_str(), "rb"); + if (file == NULL) { + Debug(Debug::ERROR) << "Cannot open dense index " << path << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + + Header header; + if (fread(&header, sizeof(header), 1, file) != 1) { + Debug(Debug::ERROR) << "Cannot read dense index header " << path << "\n"; + EXIT(EXIT_FAILURE); + } + fclose(file); + + if (header.magic != MAGIC) { + Debug(Debug::ERROR) << "File " << path << " is not a dense index\n"; + EXIT(EXIT_FAILURE); + } + if (header.version != VERSION) { + Debug(Debug::ERROR) << "Dense index " << path << " has version " << header.version + << ", this build reads version " << VERSION << "\n"; + EXIT(EXIT_FAILURE); + } + + Info info; + info.entryCount = header.entryCount; + info.firstKey = header.firstKey; + info.dataSize = header.dataSize; + info.maxSeqLen = header.maxSeqLen; + return info; +} + +DBReader::Index *DenseIndex::loadRange(const std::string &dbName, DBKeyType keyFrom, + DBKeyType keyTo, Info *rangeInfo) { + const Info whole = readInfo(dbName); + + if (keyFrom < whole.firstKey || keyTo < keyFrom + || keyTo > whole.firstKey + whole.entryCount) { + Debug(Debug::ERROR) << "Key range [" << keyFrom << ", " << keyTo << ") is outside " + << dbName << ", which holds keys [" << whole.firstKey << ", " + << (whole.firstKey + whole.entryCount) << ")\n"; + EXIT(EXIT_FAILURE); + } + + const size_t count = static_cast(keyTo - keyFrom); + const size_t rowFrom = static_cast(keyFrom - whole.firstKey); + + const std::string path = fileName(dbName); + FILE *file = fopen(path.c_str(), "rb"); + if (file == NULL) { + Debug(Debug::ERROR) << "Cannot open dense index " << path << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + + DBReader::Index *index = new DBReader::Index[count == 0 ? 1 : count]; + + uint64_t dataSize = 0; + uint32_t maxSeqLen = 0; + + // Read in chunks rather than one record at a time so a large range costs a + // handful of sequential reads instead of one syscall per key. + const size_t bufferCapacity = 65536; + Entry *buffer = new Entry[bufferCapacity]; + size_t done = 0; + while (done < count) { + const size_t batch = std::min(bufferCapacity, count - done); + const long offset = static_cast(sizeof(Header) + (rowFrom + done) * sizeof(Entry)); + if (fseek(file, offset, SEEK_SET) != 0 + || fread(buffer, sizeof(Entry), batch, file) != batch) { + Debug(Debug::ERROR) << "Cannot read dense index " << path << " at row " + << (rowFrom + done) << "\n"; + EXIT(EXIT_FAILURE); + } + for (size_t i = 0; i < batch; i++) { + index[done + i].id = static_cast(keyFrom + done + i); + index[done + i].offset = buffer[i].offset; + index[done + i].length = buffer[i].length; + dataSize += buffer[i].length; + maxSeqLen = std::max(maxSeqLen, buffer[i].length); + } + done += batch; + } + delete[] buffer; + fclose(file); + + if (rangeInfo != NULL) { + rangeInfo->entryCount = count; + rangeInfo->firstKey = keyFrom; + rangeInfo->dataSize = dataSize; + rangeInfo->maxSeqLen = maxSeqLen; + } + return index; +} + +size_t DenseIndex::entryOffset(uint64_t row) { + return sizeof(Header) + row * sizeof(Entry); +} + +void DenseIndex::createEmpty(const std::string &dbName, uint64_t entryCount, uint64_t firstKey, + uint64_t dataSize, uint32_t maxSeqLen) { + const std::string path = fileName(dbName); + FILE *file = fopen(path.c_str(), "wb"); + if (file == NULL) { + Debug(Debug::ERROR) << "Cannot create dense index " << path << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + + Header header; + memset(&header, 0, sizeof(header)); + header.magic = MAGIC; + header.version = VERSION; + header.entryCount = entryCount; + header.firstKey = firstKey; + header.dataSize = dataSize; + header.maxSeqLen = maxSeqLen; + if (fwrite(&header, sizeof(header), 1, file) != 1) { + Debug(Debug::ERROR) << "Cannot write dense index header " << path << "\n"; + EXIT(EXIT_FAILURE); + } + if (fflush(file) != 0) { + Debug(Debug::ERROR) << "Cannot flush dense index " << path << "\n"; + EXIT(EXIT_FAILURE); + } + // Size the file up front so the workers' pwrites land inside it rather than + // each extending a sparse file from a different direction. + if (ftruncate(fileno(file), static_cast(entryOffset(entryCount))) != 0) { + Debug(Debug::ERROR) << "Cannot size dense index " << path << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + if (fclose(file) != 0) { + Debug(Debug::ERROR) << "Cannot close dense index " << path << "\n"; + EXIT(EXIT_FAILURE); + } +} + +void DenseIndex::writeTextIndex(const std::string &dbName) { + const Info info = readInfo(dbName); + const std::string path = fileName(dbName); + FILE *in = fopen(path.c_str(), "rb"); + if (in == NULL) { + Debug(Debug::ERROR) << "Cannot open dense index " << path << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + if (fseek(in, static_cast(entryOffset(0)), SEEK_SET) != 0) { + Debug(Debug::ERROR) << "Cannot seek dense index " << path << "\n"; + EXIT(EXIT_FAILURE); + } + + const std::string textIndexName = dbName + ".index"; + FILE *out = FileUtil::openAndDelete(textIndexName.c_str(), "w"); + setvbuf(out, NULL, _IOFBF, 1024 * 1024 * 4); + + const size_t bufferCapacity = 65536; + Entry *buffer = new Entry[bufferCapacity]; + char line[128]; + uint64_t done = 0; + while (done < info.entryCount) { + const size_t batch = std::min(bufferCapacity, static_cast(info.entryCount - done)); + if (fread(buffer, sizeof(Entry), batch, in) != batch) { + Debug(Debug::ERROR) << "Cannot read dense index " << path << " at row " << done << "\n"; + EXIT(EXIT_FAILURE); + } + for (size_t i = 0; i < batch; i++) { + const int written = snprintf(line, sizeof(line), "%llu\t%llu\t%u\n", + (unsigned long long) (info.firstKey + done + i), + (unsigned long long) buffer[i].offset, + buffer[i].length); + if (fwrite(line, 1, static_cast(written), out) != static_cast(written)) { + Debug(Debug::ERROR) << "Cannot write index " << textIndexName << "\n"; + EXIT(EXIT_FAILURE); + } + } + done += batch; + } + delete[] buffer; + fclose(in); + if (fclose(out) != 0) { + Debug(Debug::ERROR) << "Cannot close index " << textIndexName << "\n"; + EXIT(EXIT_FAILURE); + } +} diff --git a/src/commons/DenseIndex.h b/src/commons/DenseIndex.h new file mode 100644 index 000000000..812c0a64a --- /dev/null +++ b/src/commons/DenseIndex.h @@ -0,0 +1,108 @@ +#ifndef MMSEQS_DENSEINDEX_H +#define MMSEQS_DENSEINDEX_H + +#include "DBReader.h" +#include "IndexTypes.h" + +#include +#include + +// A fixed-width companion index that makes a *key range* of a database loadable +// without touching the rest of it. +// +// MMseqs2's native .index is text with variable-length lines, so locating the +// entries for keys [from, to) means parsing every preceding line, and DBReader +// materialises the whole thing as a 24 byte Index record per entry. At 1e12 +// sequences that is ~34 TB of text parsed into ~24 TB of RAM -- per worker. No +// node can hold it, and every distributed stage would pay it just to read its own +// slice. +// +// This companion file stores one fixed-width record per key, so a key range maps +// straight to a byte range and a worker reads only what it needs. Loading a range +// costs 12 bytes per key in that range and nothing for the rest of the database. +// +// The scheme requires keys to be *dense*: key == firstKey + row. That is what the +// length-ranked createdb produces, and it is also why the length ranking matters +// beyond ordering -- it makes the key its own array offset. Databases with sparse +// keys are unaffected and keep using the text index. +class DenseIndex { +public: + // Packed so the file is 12 bytes per entry rather than 16. At 1e12 sequences + // that difference is 4 TB of scratch, and the reads are copied into aligned + // structs anyway so unaligned access never happens. + struct __attribute__((__packed__)) Entry { + uint64_t offset; + uint32_t length; + }; + + struct Info { + uint64_t entryCount; + uint64_t firstKey; + uint64_t dataSize; + uint32_t maxSeqLen; + }; + + static std::string fileName(const std::string &dbName); + static bool exists(const std::string &dbName); + + // Streams the database's text index and writes the companion file. Runs in + // constant memory -- it never materialises the index -- so it is usable at any + // scale, including as a one-off converter for a database built by stock + // createdb. + // + // Exits if the keys are not dense. Falling back silently would leave callers + // reading a companion file whose row numbers do not mean what they think. + static void build(const std::string &dbName); + + // Creates the companion file for a database that many workers are about to + // write concurrently. The header is final from the start (the planner knows + // the totals before any sequence is written) and the rows are left zeroed; + // each worker then fills only its own rows with pwrite at entryOffset(row). + // Rows are fixed width, so those writes never overlap and need no locking. + static void createEmpty(const std::string &dbName, uint64_t entryCount, uint64_t firstKey, + uint64_t dataSize, uint32_t maxSeqLen); + + // Byte offset of a row within the companion file. + static size_t entryOffset(uint64_t row); + + // Writes the text .index from the companion file, streaming in constant + // memory. The inverse of build(), for the stock tools that still read the + // text index; the distributed stages read the companion file directly. + static void writeTextIndex(const std::string &dbName); + + static Info readInfo(const std::string &dbName); + + // Loads entries for keys [keyFrom, keyTo) into a newly allocated array that + // the caller owns. The array is laid out exactly as DBReader expects, so the + // caller can hand it to DBReader's external-index constructor together with + // setDataFile()/open() and get a reader scoped to that key range: + // + // DBReader::Index *idx = DenseIndex::loadRange(db, from, to, &info); + // DBReader reader(idx, info.entryCount, info.dataSize, + // from + info.entryCount - 1, dbType, + // info.maxSeqLen, threads); + // reader.setDataFile(db.c_str()); + // reader.open(DBReader::NOSORT); + // + // rangeInfo receives the entry count, data size and max length *of the range*, + // which are the values that constructor needs. + static DBReader::Index *loadRange(const std::string &dbName, + DBKeyType keyFrom, DBKeyType keyTo, + Info *rangeInfo); + +private: + struct __attribute__((__packed__)) Header { + uint64_t magic; + uint64_t version; + uint64_t entryCount; + uint64_t firstKey; + uint64_t dataSize; + uint32_t maxSeqLen; + uint32_t reserved; + }; + + static const uint64_t MAGIC = 0x4d4d44454e534931ULL; // "MMDENSI1" + static const uint64_t VERSION = 1; +}; + +#endif diff --git a/src/commons/LengthRankedPlan.cpp b/src/commons/LengthRankedPlan.cpp new file mode 100644 index 000000000..069282ccb --- /dev/null +++ b/src/commons/LengthRankedPlan.cpp @@ -0,0 +1,207 @@ +#include "LengthRankedPlan.h" + +#include "Debug.h" +#include "FileUtil.h" + +#include +#include + +namespace { + +// Both files are written and read as a fixed header followed by a POD array, so +// a worker reads back exactly what the planner wrote with no parsing. +const uint64_t HIST_MAGIC = 0x4d4d4c5248495331ULL; // "MMLRHIS1" +const uint64_t PLAN_MAGIC = 0x4d4d4c52504c4e31ULL; // "MMLRPLN1" + +struct FileHeader { + uint64_t magic; + uint64_t chunkIdx; + uint64_t fileIdx; + uint64_t entryCount; + uint64_t seqCount; + uint64_t nuclVotes; + uint64_t sampleCount; + uint64_t reserved; +}; + +void writeBlock(const std::string &path, const FileHeader &header, + const void *entries, size_t entryBytes) { + // Write to a temporary and rename, so a worker that dies mid-write leaves no + // truncated file that a later reader would mistake for a complete one. + std::string tmp = path + ".tmp"; + FILE *file = FileUtil::openAndDelete(tmp.c_str(), "wb"); + if (fwrite(&header, sizeof(FileHeader), 1, file) != 1) { + Debug(Debug::ERROR) << "Cannot write header to " << tmp << "\n"; + EXIT(EXIT_FAILURE); + } + if (entryBytes > 0 && fwrite(entries, entryBytes, 1, file) != 1) { + Debug(Debug::ERROR) << "Cannot write entries to " << tmp << "\n"; + EXIT(EXIT_FAILURE); + } + if (fclose(file) != 0) { + Debug(Debug::ERROR) << "Cannot close " << tmp << "\n"; + EXIT(EXIT_FAILURE); + } + FileUtil::move(tmp.c_str(), path.c_str()); +} + +FileHeader readHeader(FILE *file, const std::string &path, uint64_t magic) { + FileHeader header; + if (fread(&header, sizeof(FileHeader), 1, file) != 1) { + Debug(Debug::ERROR) << "Cannot read header from " << path << "\n"; + EXIT(EXIT_FAILURE); + } + if (header.magic != magic) { + Debug(Debug::ERROR) << "File " << path << " is not a length-ranked plan file\n"; + EXIT(EXIT_FAILURE); + } + return header; +} + +} // namespace + +void ChunkHistogram::write(const std::string &path) const { + FileHeader header; + header.magic = HIST_MAGIC; + header.chunkIdx = chunkIdx; + header.fileIdx = fileIdx; + header.entryCount = buckets.size(); + header.seqCount = seqCount; + header.nuclVotes = nuclVotes; + header.sampleCount = sampleCount; + header.reserved = 0; + writeBlock(path, header, buckets.data(), buckets.size() * sizeof(Bucket)); +} + +ChunkHistogram ChunkHistogram::read(const std::string &path) { + FILE *file = FileUtil::openFileOrDie(path.c_str(), "rb", true); + FileHeader header = readHeader(file, path, HIST_MAGIC); + + ChunkHistogram histogram; + histogram.chunkIdx = header.chunkIdx; + histogram.fileIdx = header.fileIdx; + histogram.seqCount = header.seqCount; + histogram.nuclVotes = header.nuclVotes; + histogram.sampleCount = header.sampleCount; + histogram.buckets.resize(header.entryCount); + if (header.entryCount > 0 && + fread(histogram.buckets.data(), sizeof(Bucket), header.entryCount, file) != header.entryCount) { + Debug(Debug::ERROR) << "Cannot read buckets from " << path << "\n"; + EXIT(EXIT_FAILURE); + } + fclose(file); + return histogram; +} + +void ChunkPlan::write(const std::string &path) const { + FileHeader header; + header.magic = PLAN_MAGIC; + header.chunkIdx = chunkIdx; + header.fileIdx = fileIdx; + header.entryCount = entries.size(); + header.seqCount = 0; + header.nuclVotes = 0; + header.sampleCount = 0; + header.reserved = 0; + writeBlock(path, header, entries.data(), entries.size() * sizeof(Entry)); +} + +ChunkPlan ChunkPlan::read(const std::string &path) { + FILE *file = FileUtil::openFileOrDie(path.c_str(), "rb", true); + FileHeader header = readHeader(file, path, PLAN_MAGIC); + + ChunkPlan plan; + plan.chunkIdx = header.chunkIdx; + plan.fileIdx = header.fileIdx; + plan.entries.resize(header.entryCount); + if (header.entryCount > 0 && + fread(plan.entries.data(), sizeof(Entry), header.entryCount, file) != header.entryCount) { + Debug(Debug::ERROR) << "Cannot read entries from " << path << "\n"; + EXIT(EXIT_FAILURE); + } + fclose(file); + return plan; +} + +LengthRankedTotals buildLengthRankedPlan(std::vector &histograms, + std::vector &plans) { + // Order by chunk index so the plan never depends on the order the histograms + // happened to be collected in. + std::sort(histograms.begin(), histograms.end(), + [](const ChunkHistogram &a, const ChunkHistogram &b) { return a.chunkIdx < b.chunkIdx; }); + + plans.clear(); + plans.resize(histograms.size()); + for (size_t i = 0; i < histograms.size(); i++) { + plans[i].chunkIdx = histograms[i].chunkIdx; + plans[i].fileIdx = histograms[i].fileIdx; + } + + // Every distinct length in the input, longest first: that order is the key + // order of the finished database. + std::vector lengths; + for (size_t i = 0; i < histograms.size(); i++) { + for (size_t b = 0; b < histograms[i].buckets.size(); b++) { + lengths.push_back(histograms[i].buckets[b].length); + } + } + std::sort(lengths.begin(), lengths.end(), std::greater()); + lengths.erase(std::unique(lengths.begin(), lengths.end()), lengths.end()); + + // Cursor into each chunk's bucket list. The buckets are sorted ascending by + // length and we walk lengths descending, so each cursor moves backwards and + // the whole sweep stays linear in the number of buckets. + std::vector cursor(histograms.size()); + for (size_t i = 0; i < histograms.size(); i++) { + cursor[i] = histograms[i].buckets.size(); + } + + LengthRankedTotals totals; + uint64_t nextKey = 0; + uint64_t nextDataOffset = 0; + uint64_t nextHdrOffset = 0; + + for (size_t l = 0; l < lengths.size(); l++) { + const uint64_t length = lengths[l]; + for (size_t i = 0; i < histograms.size(); i++) { + if (cursor[i] == 0 || histograms[i].buckets[cursor[i] - 1].length != length) { + continue; + } + const ChunkHistogram::Bucket &bucket = histograms[i].buckets[cursor[i] - 1]; + cursor[i]--; + + ChunkPlan::Entry entry; + entry.length = length; + entry.count = bucket.count; + entry.keyStart = nextKey; + entry.dataOffset = nextDataOffset; + entry.hdrOffset = nextHdrOffset; + plans[i].entries.push_back(entry); + + nextKey += bucket.count; + // A sequence of length L always occupies L + 2 data bytes, so this + // stays exact without looking at the sequences themselves. + nextDataOffset += bucket.count * (length + 2); + nextHdrOffset += bucket.headerBytes; + } + } + + for (size_t i = 0; i < histograms.size(); i++) { + if (cursor[i] != 0) { + Debug(Debug::ERROR) << "Chunk " << histograms[i].chunkIdx + << " has buckets that are not sorted by length\n"; + EXIT(EXIT_FAILURE); + } + // Written ascending by length; pass 2 looks entries up by length, and a + // sorted list keeps that a binary search. + std::reverse(plans[i].entries.begin(), plans[i].entries.end()); + totals.nuclVotes += histograms[i].nuclVotes; + totals.sampleCount += histograms[i].sampleCount; + } + + totals.seqCount = nextKey; + totals.dataBytes = nextDataOffset; + totals.headerBytes = nextHdrOffset; + totals.maxSeqLen = lengths.empty() ? 0 : lengths[0]; + return totals; +} diff --git a/src/commons/LengthRankedPlan.h b/src/commons/LengthRankedPlan.h new file mode 100644 index 000000000..54264bc89 --- /dev/null +++ b/src/commons/LengthRankedPlan.h @@ -0,0 +1,116 @@ +#ifndef MMSEQS_LENGTHRANKEDPLAN_H +#define MMSEQS_LENGTHRANKEDPLAN_H + +#include +#include +#include + +// Placement plan for a length-ranked, densely-keyed sequence database that many +// independent workers build into one set of output files without ever merging. +// +// The distributed createdb runs two passes over the input FASTA. Pass 1 has each +// worker scan a byte range (a "chunk") and report only a histogram: for every +// sequence length in that chunk, how many sequences have it and how many bytes +// their headers occupy. Pass 2 has each worker rescan its chunk and write the +// sequences straight into their final position in the output files. +// +// What makes pass 2 possible with no merge step is that the histograms alone +// determine every byte offset in the finished database: +// +// - keys are assigned longest sequence first, so all sequences of one length +// occupy a contiguous key range whose start is simply the number of longer +// sequences in the whole input; +// - a sequence of length L occupies exactly L + 2 bytes of the data file (the +// residues, a newline, and the NUL terminator DBWriter appends), so a length +// bucket's data offset is a weighted sum over the longer buckets; +// - header lengths vary per sequence, but pass 1 already totalled them per +// (chunk, length) -- exactly the granularity at which offsets are needed. +// +// Ties are broken by chunk index and then by position within the chunk, which is +// input order. The key assignment therefore depends only on the input and the +// chunk size, never on how many workers ran or in what order they finished. That +// is what makes the build both restartable and topology-invariant. +// +// The whole scheme rests on keys being dense and length-ordered, which is also +// what lets DenseIndex address an entry by key and what makes the distributed +// greedy in the reduce stage exact. + +// What one worker reports after scanning one chunk in pass 1. +class ChunkHistogram { +public: + // One sequence length present in the chunk. Sorted ascending by length. + struct Bucket { + uint64_t length; + uint64_t count; + // Total bytes the headers of those sequences occupy in the header + // database, i.e. the header text including its newline plus the NUL + // that DBWriter appends. Summed here because header size cannot be + // derived from sequence length the way data size can. + uint64_t headerBytes; + }; + + uint64_t chunkIdx; + // Index into the sorted input file list. Needed to reconstruct which source + // file every key came from when the .lookup file is written. + uint64_t fileIdx; + uint64_t seqCount; + // Nucleotide detection votes, aggregated across chunks by the planner so the + // database type is decided from the whole input rather than one chunk. + uint64_t nuclVotes; + uint64_t sampleCount; + std::vector buckets; + + ChunkHistogram() : chunkIdx(0), fileIdx(0), seqCount(0), nuclVotes(0), sampleCount(0) {} + + void write(const std::string &path) const; + // Exits if the file is missing or malformed: a chunk histogram that cannot be + // read means the plan would silently place sequences at wrong offsets. + static ChunkHistogram read(const std::string &path); +}; + +// Where one worker must put every sequence of its chunk in pass 2. +class ChunkPlan { +public: + struct Entry { + uint64_t length; + uint64_t count; + // First key, and first data/header byte, of this chunk's run of + // sequences of this length. The worker consumes them in input order, + // advancing each cursor as it writes. + uint64_t keyStart; + uint64_t dataOffset; + uint64_t hdrOffset; + }; + + uint64_t chunkIdx; + uint64_t fileIdx; + std::vector entries; + + ChunkPlan() : chunkIdx(0), fileIdx(0) {} + + void write(const std::string &path) const; + static ChunkPlan read(const std::string &path); +}; + +// Totals for the finished database, produced together with the per-chunk plans. +struct LengthRankedTotals { + uint64_t seqCount; + uint64_t dataBytes; + uint64_t headerBytes; + uint64_t maxSeqLen; + uint64_t nuclVotes; + uint64_t sampleCount; + + LengthRankedTotals() + : seqCount(0), dataBytes(0), headerBytes(0), maxSeqLen(0), nuclVotes(0), sampleCount(0) {} +}; + +// Turns the per-chunk histograms into per-chunk placement plans. +// +// Input histograms may arrive in any order; they are ordered by chunkIdx here so +// the result does not depend on directory listing order. plans is resized to one +// entry per histogram, indexed the same way as the sorted chunk order. +LengthRankedTotals buildLengthRankedPlan(std::vector &histograms, + std::vector &plans); + +#endif diff --git a/src/commons/ParallelCoordination.cpp b/src/commons/ParallelCoordination.cpp new file mode 100644 index 000000000..619d620dd --- /dev/null +++ b/src/commons/ParallelCoordination.cpp @@ -0,0 +1,393 @@ +#include "ParallelCoordination.h" + +#include "Debug.h" +#include "Util.h" + +#include +#include +#include +#include + +#include +#include + +namespace { + +// Retries around EINTR, which a long F_SETLKW wait or a large pread can hit when +// the process takes a signal (Slurm sends plenty). +ssize_t preadFully(int fd, void *buffer, size_t size, off_t offset) { + char *out = static_cast(buffer); + size_t done = 0; + while (done < size) { + ssize_t got = pread(fd, out + done, size - done, offset + static_cast(done)); + if (got < 0) { + if (errno == EINTR) { + continue; + } + return -1; + } + if (got == 0) { + break; + } + done += static_cast(got); + } + return static_cast(done); +} + +ssize_t pwriteFully(int fd, const void *buffer, size_t size, off_t offset) { + const char *in = static_cast(buffer); + size_t done = 0; + while (done < size) { + ssize_t put = pwrite(fd, in + done, size - done, offset + static_cast(done)); + if (put < 0) { + if (errno == EINTR) { + continue; + } + return -1; + } + done += static_cast(put); + } + return static_cast(done); +} + +int64_t nowSeconds() { + return static_cast(time(NULL)); +} + +} // namespace + +FileLock::FileLock(const std::string &path) : path(path), fd(-1) { + fd = open(path.c_str(), O_RDWR | O_CREAT, 0666); + if (fd < 0) { + Debug(Debug::ERROR) << "Could not open coordination file " << path << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } +} + +FileLock::~FileLock() { + if (fd >= 0) { + close(fd); + } +} + +void FileLock::lock() { + threadMutex.lock(); + + struct flock request; + memset(&request, 0, sizeof(request)); + request.l_type = F_WRLCK; + request.l_whence = SEEK_SET; + request.l_start = 0; + // len 0 means "to end of file, including anything appended later". + request.l_len = 0; + + while (fcntl(fd, F_SETLKW, &request) < 0) { + if (errno == EINTR) { + continue; + } + Debug(Debug::ERROR) << "Could not lock coordination file " << path << ": " + << strerror(errno) << "\n"; + // A failure here means the filesystem is not honouring fcntl locks (a + // Lustre mount without -o flock is the usual cause). Continuing would + // silently corrupt shared state, so stop instead. + threadMutex.unlock(); + EXIT(EXIT_FAILURE); + } +} + +void FileLock::unlock() { + struct flock request; + memset(&request, 0, sizeof(request)); + request.l_type = F_UNLCK; + request.l_whence = SEEK_SET; + request.l_start = 0; + request.l_len = 0; + + while (fcntl(fd, F_SETLK, &request) < 0) { + if (errno == EINTR) { + continue; + } + Debug(Debug::ERROR) << "Could not unlock coordination file " << path << ": " + << strerror(errno) << "\n"; + break; + } + + threadMutex.unlock(); +} + +SharedCounter::SharedCounter(const std::string &path) : lock(path) { +} + +int64_t SharedCounter::readLocked() { + int64_t value = 0; + ssize_t got = preadFully(lock.getFd(), &value, sizeof(value), 0); + if (got < 0) { + Debug(Debug::ERROR) << "Could not read shared counter: " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + // A short read means the file was just created and is still empty, which is + // the same as a counter of zero. + if (got != static_cast(sizeof(value))) { + return 0; + } + return value; +} + +void SharedCounter::writeLocked(int64_t value) { + if (pwriteFully(lock.getFd(), &value, sizeof(value), 0) < 0) { + Debug(Debug::ERROR) << "Could not write shared counter: " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + // Push the update out before releasing the lock, so a reader on another node + // that acquires the lock next is guaranteed to observe it. + fsync(lock.getFd()); +} + +int64_t SharedCounter::fetchAdd(int64_t n) { + lock.lock(); + int64_t previous = readLocked(); + writeLocked(previous + n); + lock.unlock(); + return previous; +} + +int64_t SharedCounter::get() { + lock.lock(); + int64_t value = readLocked(); + lock.unlock(); + return value; +} + +void SharedCounter::await(int64_t target, unsigned int pollSeconds) { + while (get() < target) { + sleep(pollSeconds); + } +} + +WorkQueue::WorkQueue(const std::string &path, int64_t itemCount) + : path(path), itemCount(itemCount), lock(path) { + lock.lock(); + initialiseLocked(); + lock.unlock(); +} + +void WorkQueue::initialiseLocked() { + Header header; + ssize_t got = preadFully(lock.getFd(), &header, sizeof(header), 0); + if (got < 0) { + Debug(Debug::ERROR) << "Could not read work queue " << path << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + + if (got == static_cast(sizeof(header)) && header.magic == MAGIC) { + if (header.version != VERSION) { + Debug(Debug::ERROR) << "Work queue " << path << " has version " << header.version + << ", this build writes version " << VERSION << "\n"; + EXIT(EXIT_FAILURE); + } + if (header.itemCount != static_cast(itemCount)) { + // Resuming a run that partitioned its work differently would silently + // skip or duplicate items, so refuse rather than guess. + Debug(Debug::ERROR) << "Work queue " << path << " was created for " + << header.itemCount << " items but this run has " << itemCount + << ". Remove the coordination directory to start over.\n"; + EXIT(EXIT_FAILURE); + } + return; + } + + Header fresh; + memset(&fresh, 0, sizeof(fresh)); + fresh.magic = MAGIC; + fresh.version = VERSION; + fresh.itemCount = static_cast(itemCount); + fresh.doneCount = 0; + fresh.nextHint = 0; + writeHeaderLocked(fresh); + + // Zero-fill the record array so every later access is a plain offset read + // rather than a short read off the end of a sparse file. PENDING is state 0, + // so zeroing is also the correct initial state. + const size_t batchSize = 65536; + Record *blank = new Record[batchSize]; + memset(blank, 0, batchSize * sizeof(Record)); + for (int64_t written = 0; written < itemCount; written += static_cast(batchSize)) { + size_t count = static_cast(std::min(batchSize, itemCount - written)); + if (pwriteFully(lock.getFd(), blank, count * sizeof(Record), recordOffset(written)) < 0) { + Debug(Debug::ERROR) << "Could not initialise work queue " << path << ": " + << strerror(errno) << "\n"; + delete[] blank; + EXIT(EXIT_FAILURE); + } + } + delete[] blank; + fsync(lock.getFd()); +} + +WorkQueue::Header WorkQueue::readHeaderLocked() { + Header header; + if (preadFully(lock.getFd(), &header, sizeof(header), 0) != static_cast(sizeof(header))) { + Debug(Debug::ERROR) << "Could not read work queue header " << path << "\n"; + EXIT(EXIT_FAILURE); + } + return header; +} + +void WorkQueue::writeHeaderLocked(const Header &header) { + if (pwriteFully(lock.getFd(), &header, sizeof(header), 0) < 0) { + Debug(Debug::ERROR) << "Could not write work queue header " << path << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } +} + +WorkQueue::Record WorkQueue::readRecordLocked(int64_t index) { + Record record; + if (preadFully(lock.getFd(), &record, sizeof(record), recordOffset(index)) + != static_cast(sizeof(record))) { + Debug(Debug::ERROR) << "Could not read work queue record " << index << " in " << path << "\n"; + EXIT(EXIT_FAILURE); + } + return record; +} + +void WorkQueue::writeRecordLocked(int64_t index, const Record &record) { + if (pwriteFully(lock.getFd(), &record, sizeof(record), recordOffset(index)) < 0) { + Debug(Debug::ERROR) << "Could not write work queue record " << index << " in " << path + << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } +} + +int64_t WorkQueue::claim(int64_t workerId, int64_t leaseSeconds) { + lock.lock(); + + Header header = readHeaderLocked(); + const int64_t now = nowSeconds(); + + int64_t firstUnfinished = -1; + for (int64_t index = static_cast(header.nextHint); index < itemCount; index++) { + Record record = readRecordLocked(index); + if (record.state == DONE) { + continue; + } + if (firstUnfinished < 0) { + firstUnfinished = index; + } + // Untouched, or held by a worker whose lease ran out -- in both cases the + // item is ours to take. Re-claiming after a lease expiry is what makes a + // killed job recoverable without an operator deciding what to redo. + if (record.state == PENDING + || (record.state == CLAIMED && static_cast(record.leaseExpiry) <= now)) { + record.state = CLAIMED; + record.worker = static_cast(workerId); + record.leaseExpiry = static_cast(now + leaseSeconds); + writeRecordLocked(index, record); + + if (firstUnfinished > static_cast(header.nextHint)) { + header.nextHint = static_cast(firstUnfinished); + writeHeaderLocked(header); + } + fsync(lock.getFd()); + lock.unlock(); + return index; + } + } + + if (firstUnfinished > static_cast(header.nextHint)) { + header.nextHint = static_cast(firstUnfinished); + writeHeaderLocked(header); + fsync(lock.getFd()); + } + + lock.unlock(); + return -1; +} + +void WorkQueue::completeLocked(int64_t index, int64_t workerId) { + Record record = readRecordLocked(index); + if (record.state == DONE) { + // Already recorded, either by us before a crash or by another worker that + // re-claimed the item after our lease lapsed. Nothing to do; making this + // idempotent is what lets a worker redo an item it may or may not have + // finished before it died. + return; + } + + record.state = DONE; + record.worker = static_cast(workerId); + record.leaseExpiry = 0; + writeRecordLocked(index, record); + + Header header = readHeaderLocked(); + header.doneCount += 1; + writeHeaderLocked(header); + fsync(lock.getFd()); +} + +void WorkQueue::renew(int64_t index, int64_t workerId, int64_t leaseSeconds) { + lock.lock(); + Record record = readRecordLocked(index); + // Only extend a claim we still hold. If the lease already lapsed and someone + // else took the item, stealing it back would run it twice. + if (record.state == CLAIMED && record.worker == static_cast(workerId)) { + record.leaseExpiry = static_cast(nowSeconds() + leaseSeconds); + writeRecordLocked(index, record); + fsync(lock.getFd()); + } + lock.unlock(); +} + +void WorkQueue::complete(int64_t index, int64_t workerId) { + lock.lock(); + completeLocked(index, workerId); + lock.unlock(); +} + +void WorkQueue::release(int64_t index, int64_t workerId) { + lock.lock(); + Record record = readRecordLocked(index); + if (record.state == CLAIMED && record.worker == static_cast(workerId)) { + record.state = PENDING; + record.worker = 0; + record.leaseExpiry = 0; + writeRecordLocked(index, record); + fsync(lock.getFd()); + } + lock.unlock(); +} + +int64_t WorkQueue::getDoneCount() { + lock.lock(); + Header header = readHeaderLocked(); + lock.unlock(); + return static_cast(header.doneCount); +} + +bool WorkQueue::allDone() { + return getDoneCount() >= itemCount; +} + +bool WorkQueue::awaitAll(unsigned int pollSeconds, unsigned int stallSeconds) { + int64_t lastDone = -1; + int64_t lastProgress = nowSeconds(); + while (true) { + int64_t done = getDoneCount(); + if (done >= itemCount) { + return true; + } + if (done != lastDone) { + lastDone = done; + lastProgress = nowSeconds(); + } else if (stallSeconds > 0 + && nowSeconds() - lastProgress > static_cast(stallSeconds)) { + Debug(Debug::WARNING) << "Work queue " << path << " made no progress for " + << stallSeconds << " s (" << done << "/" << itemCount + << " done)\n"; + return false; + } + sleep(pollSeconds); + } +} diff --git a/src/commons/ParallelCoordination.h b/src/commons/ParallelCoordination.h new file mode 100644 index 000000000..4a7308df7 --- /dev/null +++ b/src/commons/ParallelCoordination.h @@ -0,0 +1,178 @@ +#ifndef MMSEQS_PARALLELCOORDINATION_H +#define MMSEQS_PARALLELCOORDINATION_H + +#include +#include +#include + +// Shared-filesystem coordination for multi-node MMseqs2 stages. +// +// Every worker process runs the *same* command line and coordinates only through +// files in a shared directory: there is no MPI, no rank argument, and no direct +// node-to-node communication. Workers may join late, die, and be restarted; the +// on-disk state is the single source of truth. +// +// All mutual exclusion uses POSIX fcntl() whole-file write locks. That choice is +// deliberate: fcntl locks are honoured cluster-wide on GPFS, Lustre (mounted with +// -o flock) and NFSv4, whereas flock() is node-local on several of them and would +// silently give no protection at all. Two consequences of fcntl semantics drive +// the implementation below: +// - locks are owned by the *process*, not the thread, so a process-local mutex +// is needed as well to serialise threads within one worker; +// - locks are dropped when *any* file descriptor to the file is closed, so the +// descriptor is opened once and owned for the object's lifetime. +// Locks are released automatically when a process dies, so a crashed worker can +// never deadlock the run. +class FileLock { +public: + // Opens (creating if needed) the lock file and keeps the descriptor for the + // object's lifetime. Does not acquire the lock. + explicit FileLock(const std::string &path); + ~FileLock(); + + // Blocks until the exclusive lock is held. Acquires the process-local mutex + // first so that concurrent threads in this process serialise against each + // other rather than all believing they hold the (per-process) fcntl lock. + void lock(); + void unlock(); + + int getFd() const { return fd; } + +private: + FileLock(const FileLock &); + FileLock &operator=(const FileLock &); + + std::string path; + int fd; + std::mutex threadMutex; +}; + +// A single 64-bit integer in a shared file, updated atomically across nodes. +// +// Used for worker-id assignment (every worker calls fetchAdd once at startup and +// takes the returned value as its identity, so identities stay unique without any +// central authority) and as a completion counter for barriers. +class SharedCounter { +public: + explicit SharedCounter(const std::string &path); + + // Adds n and returns the value *before* the addition. + int64_t fetchAdd(int64_t n = 1); + int64_t get(); + + // Polls until the counter reaches at least target. Intended for coarse + // stage barriers; callers that need failure detection should use WorkQueue, + // which tracks leases, rather than blocking here indefinitely. + void await(int64_t target, unsigned int pollSeconds = 1); + +private: + int64_t readLocked(); + void writeLocked(int64_t value); + + FileLock lock; +}; + +// A crash-tolerant work queue over a fixed set of items numbered [0, itemCount). +// +// Items are claimed under a lease rather than simply popped. A worker that dies +// holding a claim has its lease expire, after which another worker re-claims the +// item. This is the property a 24 h walltime makes mandatory: a stage that +// outlives its Slurm job must be resumable by a fresh set of workers with no +// manual intervention and no lost or duplicated work. +// +// The queue lives in one file: a fixed-size header followed by one fixed-width +// record per item. Fixed widths mean claiming never rewrites the whole file, and +// state is read back by offset rather than by scanning lines. +class WorkQueue { +public: + enum State { + PENDING = 0, + CLAIMED = 1, + DONE = 2 + }; + + // Opens the queue at path, creating it for itemCount items if it does not + // exist. Safe to call concurrently from every worker: the first to get the + // lock initialises the file and the rest observe the finished state. + // + // Re-opening an existing queue keeps its progress, which is what makes a + // stage resumable across jobs. Re-opening with a different itemCount is a + // programming error and exits, since it would mean the work was partitioned + // differently than in the run being resumed. + WorkQueue(const std::string &path, int64_t itemCount); + + // Claims the lowest-numbered item that is either untouched or whose holder's + // lease has expired. Returns the item index, or -1 when no item is currently + // claimable (either everything is done, or every remaining item is held by a + // worker with a live lease). + int64_t claim(int64_t workerId, int64_t leaseSeconds = DEFAULT_LEASE_SECONDS); + + // Extends the lease on a held item. Long-running items must call this more + // often than leaseSeconds, otherwise another worker will treat the item as + // abandoned and start it again. + void renew(int64_t index, int64_t workerId, int64_t leaseSeconds = DEFAULT_LEASE_SECONDS); + + // Marks a held item finished. Idempotent, so a worker that completes an item + // and dies before recording it can safely redo and re-complete it. + void complete(int64_t index, int64_t workerId); + + // Releases a held item back to PENDING without completing it, so it is + // re-claimable immediately rather than after the lease expires. + void release(int64_t index, int64_t workerId); + + int64_t getItemCount() const { return itemCount; } + int64_t getDoneCount(); + bool allDone(); + + // Polls until every item is DONE. Returns false if no progress was made and + // no item was claimable for stallSeconds, which means the remaining work is + // held by workers that are gone but whose leases have not yet expired, or + // that the run is genuinely stuck. + bool awaitAll(unsigned int pollSeconds = 5, unsigned int stallSeconds = 0); + + static const int64_t DEFAULT_LEASE_SECONDS = 1800; + +private: + // On-disk layout. Both structs are written and read verbatim; sizes are + // asserted at construction so a mismatched build cannot corrupt a queue + // written by another binary. + struct Header { + uint64_t magic; + uint64_t version; + uint64_t itemCount; + uint64_t doneCount; + // Lowest index that might not be DONE yet. Purely an optimisation: it + // lets claim() skip a finished prefix instead of rescanning from 0. + uint64_t nextHint; + uint64_t reserved[3]; + }; + + struct Record { + uint32_t state; + uint32_t worker; + // Unix seconds after which the claim is considered abandoned. Node + // clocks only need to agree to within a fraction of the lease, which + // NTP-synchronised HPC nodes comfortably do. + uint64_t leaseExpiry; + }; + + static const uint64_t MAGIC = 0x4d4d51554555453fULL; // "MMQUEUE?" + static const uint64_t VERSION = 1; + + void initialiseLocked(); + Header readHeaderLocked(); + void writeHeaderLocked(const Header &header); + Record readRecordLocked(int64_t index); + void writeRecordLocked(int64_t index, const Record &record); + void completeLocked(int64_t index, int64_t workerId); + + static size_t recordOffset(int64_t index) { + return sizeof(Header) + static_cast(index) * sizeof(Record); + } + + std::string path; + int64_t itemCount; + FileLock lock; +}; + +#endif diff --git a/src/commons/Parameters.cpp b/src/commons/Parameters.cpp index 351a81fcc..f113406cc 100644 --- a/src/commons/Parameters.cpp +++ b/src/commons/Parameters.cpp @@ -55,6 +55,7 @@ Parameters::Parameters(): PARAM_SPLIT(PARAM_SPLIT_ID, "--split", "Split database", "Split input into N equally distributed chunks. 0: set the best split automatically", typeid(int), (void *) &split, "^[0-9]{1}[0-9]*$", MMseqsParameter::COMMAND_PREFILTER | MMseqsParameter::COMMAND_EXPERT), PARAM_SPLIT_MODE(PARAM_SPLIT_MODE_ID, "--split-mode", "Split mode", "0: split target db; 1: split query db; 2: auto, depending on main memory", typeid(int), (void *) &splitMode, "^[0-2]{1}$", MMseqsParameter::COMMAND_PREFILTER | MMseqsParameter::COMMAND_EXPERT), PARAM_SPLIT_MEMORY_LIMIT(PARAM_SPLIT_MEMORY_LIMIT_ID, "--split-memory-limit", "Split memory limit", "Set max memory per split. E.g. 800B, 5K, 10M, 1G. Default (0) to all available system memory", typeid(ByteParser), (void *) &splitMemoryLimit, "^(0|[1-9]{1}[0-9]*(B|K|M|G|T)?)$", MMseqsParameter::COMMAND_COMMON | MMseqsParameter::COMMAND_PREFILTER | MMseqsParameter::COMMAND_EXPERT), + PARAM_CHUNK_SIZE(PARAM_CHUNK_SIZE_ID, "--chunk-size", "Chunk size", "Input bytes one work item covers. Smaller chunks spread work more evenly and use less memory per thread; larger chunks mean fewer coordination files. E.g. 64M, 256M, 1G", typeid(ByteParser), (void *) &chunkSize, "^([1-9]{1}[0-9]*(B|K|M|G|T)?)$", MMseqsParameter::COMMAND_EXPERT), PARAM_DISK_SPACE_LIMIT(PARAM_DISK_SPACE_LIMIT_ID, "--disk-space-limit", "Disk space limit", "Set max disk space to use for reverse profile searches. E.g. 800B, 5K, 10M, 1G. Default (0) to all available disk space in the temp folder", typeid(ByteParser), (void *) &diskSpaceLimit, "^(0|[1-9]{1}[0-9]*(B|K|M|G|T)?)$", MMseqsParameter::COMMAND_COMMON | MMseqsParameter::COMMAND_PREFILTER | MMseqsParameter::COMMAND_EXPERT), PARAM_SPLIT_AMINOACID(PARAM_SPLIT_AMINOACID_ID, "--split-aa", "Split by amino acid", "Try to find the best split boundaries by entry lengths", typeid(bool), (void *) &splitAA, "$", MMseqsParameter::COMMAND_EXPERT), PARAM_SUB_MAT(PARAM_SUB_MAT_ID, "--sub-mat", "Substitution matrix", "Substitution matrix file", typeid(MultiParam>), (void *) &scoringMatrixFile, "", MMseqsParameter::COMMAND_COMMON | MMseqsParameter::COMMAND_EXPERT), @@ -909,6 +910,16 @@ Parameters::Parameters(): createdb.push_back(&PARAM_GPU); createdb.push_back(&PARAM_V); + // createdbparallel + // No PARAM_WRITE_LOOKUP: .lookup is not emitted yet. It is variable-width and + // in key order, so it cannot be written at computed offsets like everything + // else; see PARALLEL_LINCLUST_WIP.md for the reconstruction sketch. + createdbparallel.push_back(&PARAM_DB_TYPE); + createdbparallel.push_back(&PARAM_CHUNK_SIZE); + createdbparallel.push_back(&PARAM_THREADS); + createdbparallel.push_back(&PARAM_COMPRESSED); + createdbparallel.push_back(&PARAM_V); + // makepaddedseqdb makepaddedseqdb.push_back(&PARAM_SUB_MAT); makepaddedseqdb.push_back(&PARAM_SCORE_BIAS); @@ -2499,6 +2510,7 @@ void Parameters::setDefaults() { split = AUTO_SPLIT_DETECTION; splitMode = DETECT_BEST_DB_SPLIT; splitMemoryLimit = 0; + chunkSize = 256 * 1024 * 1024; diskSpaceLimit = 0; splitAA = false; spacedKmerPattern = ""; diff --git a/src/commons/Parameters.h b/src/commons/Parameters.h index 0b0837808..18d0b5b0f 100644 --- a/src/commons/Parameters.h +++ b/src/commons/Parameters.h @@ -439,6 +439,7 @@ class Parameters { int split; // Split database in n equal chunks int splitMode; // Split by query or target DB size_t splitMemoryLimit; // Maximum memory in bytes a split can use + size_t chunkSize; // Input bytes one createdbparallel work item covers size_t diskSpaceLimit; // Maximum disk space in bytes for sliced reverse profile search bool splitAA; // Split database by amino acid count instead int preloadMode; // Preload mode of database @@ -827,6 +828,7 @@ class Parameters { PARAMETER(PARAM_SPLIT) PARAMETER(PARAM_SPLIT_MODE) PARAMETER(PARAM_SPLIT_MEMORY_LIMIT) + PARAMETER(PARAM_CHUNK_SIZE) PARAMETER(PARAM_DISK_SPACE_LIMIT) PARAMETER(PARAM_SPLIT_AMINOACID) PARAMETER(PARAM_SUB_MAT) @@ -1232,6 +1234,7 @@ class Parameters { std::vector createlinindex; std::vector convertalignments; std::vector createdb; + std::vector createdbparallel; std::vector makepaddedseqdb; std::vector convert2fasta; std::vector result2flat; diff --git a/src/linclust/CMakeLists.txt b/src/linclust/CMakeLists.txt index 527c292ab..2dbdf23ae 100644 --- a/src/linclust/CMakeLists.txt +++ b/src/linclust/CMakeLists.txt @@ -1,5 +1,6 @@ set(linclust_source_files linclust/kmermatcher.cpp + linclust/KmerPartition.cpp linclust/kmerindexdb.cpp linclust/kmersearch.cpp linclust/LinsearchIndexReader.cpp diff --git a/src/linclust/KmerPartition.cpp b/src/linclust/KmerPartition.cpp new file mode 100644 index 000000000..2516d47a2 --- /dev/null +++ b/src/linclust/KmerPartition.cpp @@ -0,0 +1,174 @@ +#include "KmerPartition.h" + +#include "Debug.h" +#include "FileUtil.h" +#include "Util.h" + +#include +#include +#include + +#include +#include + +KmerPartitioner::KmerPartitioner(unsigned int partitionCount) : partitionCount(partitionCount) { + if (partitionCount == 0 || (partitionCount & (partitionCount - 1)) != 0) { + Debug(Debug::ERROR) << "Partition count " << partitionCount << " is not a power of two\n"; + EXIT(EXIT_FAILURE); + } + if (partitionCount > 65536) { + // The score being partitioned is 16 bits, so more partitions than that + // would leave some permanently empty and silently unbalance the run. + Debug(Debug::ERROR) << "Partition count " << partitionCount + << " exceeds the 16-bit k-mer hash space (max 65536)\n"; + EXIT(EXIT_FAILURE); + } + unsigned int bits = 0; + while ((1u << bits) < partitionCount) { + bits++; + } + shift = 16 - bits; +} + +std::string KmerBucketWriter::partitionDir(const std::string &dir, unsigned int partition) { + return dir + "/p" + SSTR(partition); +} + +void KmerBucketWriter::createLayout(const std::string &dir, unsigned int partitionCount) { + if (FileUtil::directoryExists(dir.c_str()) == false) { + FileUtil::makeDir(dir.c_str()); + } + for (unsigned int p = 0; p < partitionCount; p++) { + const std::string path = partitionDir(dir, p); + if (FileUtil::directoryExists(path.c_str()) == false) { + // Racing workers may both try this; only a failure that also leaves + // no directory behind is real. + if (mkdir(path.c_str(), 0777) != 0 && FileUtil::directoryExists(path.c_str()) == false) { + Debug(Debug::ERROR) << "Cannot create bucket directory " << path << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + } + } +} + +KmerBucketWriter::KmerBucketWriter(const std::string &dir, unsigned int partitionCount, + const std::string &shardId, size_t bufferBudgetBytes) + : dir(dir), shardId(shardId), partitionCount(partitionCount), recordCount(0) { + // At least a handful of records per partition even with a tiny budget, so a + // large partition count degrades to more frequent flushes rather than to + // one write syscall per k-mer. + const size_t perPartition = bufferBudgetBytes / (partitionCount * sizeof(KmerRecord)); + recordsPerBuffer = std::max(perPartition, 16); + buffers.resize(partitionCount); + files.assign(partitionCount, NULL); +} + +KmerBucketWriter::~KmerBucketWriter() { + close(); +} + +void KmerBucketWriter::flush(unsigned int partition) { + std::vector &buffer = buffers[partition]; + if (buffer.empty()) { + return; + } + if (files[partition] == NULL) { + // Opened lazily: with 8192 partitions and a sparse shard, most buckets + // stay untouched and should not cost a file descriptor or an empty file. + const std::string path = partitionDir(dir, partition) + "/" + shardId + ".kmers"; + files[partition] = fopen(path.c_str(), "wb"); + if (files[partition] == NULL) { + Debug(Debug::ERROR) << "Cannot open bucket " << path << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + } + if (fwrite(buffer.data(), sizeof(KmerRecord), buffer.size(), files[partition]) != buffer.size()) { + Debug(Debug::ERROR) << "Cannot write k-mer bucket for partition " << partition << "\n"; + EXIT(EXIT_FAILURE); + } + buffer.clear(); +} + +void KmerBucketWriter::append(unsigned int partition, const KmerRecord &record) { + buffers[partition].push_back(record); + if (buffers[partition].size() >= recordsPerBuffer) { + flush(partition); + } + recordCount++; +} + +void KmerBucketWriter::close() { + for (unsigned int p = 0; p < partitionCount; p++) { + flush(p); + if (files[p] != NULL) { + if (fclose(files[p]) != 0) { + Debug(Debug::ERROR) << "Cannot close k-mer bucket for partition " << p << "\n"; + EXIT(EXIT_FAILURE); + } + files[p] = NULL; + } + } +} + +std::vector KmerBucketReader::shardFiles(const std::string &dir, unsigned int partition) { + const std::string path = KmerBucketWriter::partitionDir(dir, partition); + std::vector shards; + DIR *handle = opendir(path.c_str()); + if (handle == NULL) { + // A partition no worker wrote to is empty, not an error. + return shards; + } + struct dirent *entry; + while ((entry = readdir(handle)) != NULL) { + const std::string name = entry->d_name; + if (name.size() > 6 && name.compare(name.size() - 6, 6, ".kmers") == 0) { + shards.push_back(path + "/" + name); + } + } + closedir(handle); + // Sorted so a partition reads back in the same order on every run, which + // keeps the whole pipeline reproducible regardless of directory order. + std::sort(shards.begin(), shards.end()); + return shards; +} + +uint64_t KmerBucketReader::countRecords(const std::string &dir, unsigned int partition) { + const std::vector shards = shardFiles(dir, partition); + uint64_t total = 0; + for (size_t i = 0; i < shards.size(); i++) { + const size_t bytes = FileUtil::getFileSize(shards[i]); + if (bytes % sizeof(KmerRecord) != 0) { + Debug(Debug::ERROR) << "Bucket " << shards[i] << " is " << bytes + << " bytes, not a whole number of k-mer records. " + << "It was probably written by an interrupted worker.\n"; + EXIT(EXIT_FAILURE); + } + total += bytes / sizeof(KmerRecord); + } + return total; +} + +void KmerBucketReader::readPartition(const std::string &dir, unsigned int partition, + std::vector &out) { + const std::vector shards = shardFiles(dir, partition); + for (size_t i = 0; i < shards.size(); i++) { + const size_t bytes = FileUtil::getFileSize(shards[i]); + if (bytes % sizeof(KmerRecord) != 0) { + Debug(Debug::ERROR) << "Bucket " << shards[i] << " is truncated\n"; + EXIT(EXIT_FAILURE); + } + const size_t count = bytes / sizeof(KmerRecord); + if (count == 0) { + continue; + } + FILE *file = FileUtil::openFileOrDie(shards[i].c_str(), "rb", true); + const size_t offset = out.size(); + out.resize(offset + count); + if (fread(out.data() + offset, sizeof(KmerRecord), count, file) != count) { + Debug(Debug::ERROR) << "Cannot read bucket " << shards[i] << "\n"; + EXIT(EXIT_FAILURE); + } + fclose(file); + } +} diff --git a/src/linclust/KmerPartition.h b/src/linclust/KmerPartition.h new file mode 100644 index 000000000..7aaa9eb48 --- /dev/null +++ b/src/linclust/KmerPartition.h @@ -0,0 +1,176 @@ +#ifndef MMSEQS_KMERPARTITION_H +#define MMSEQS_KMERPARTITION_H + +#include +#include +#include +#include + +// K-mer space partitioning for the distributed linclust map/reduce. +// +// Stock kmermatcher handles a database too large for memory by *splitting*: it +// re-reads the whole database once per split and re-extracts every k-mer, then +// throws away the k-mers whose hash falls outside the split's range +// (kmermatcher.cpp:322, loop at :139-144). At 1e12 sequences on a 2 TB node that +// is ~68 splits, so the extraction work -- the single most expensive part of the +// pipeline -- is done ~68 times over. +// +// Here the database is scanned *once*. Every extracted k-mer is appended to the +// bucket file of its partition, so the extraction is paid for exactly once and +// the reduce stage later reads back one self-contained partition at a time. +// +// Why partitioning by the existing 16-bit hash is correct: kmermatcher already +// computes `score = (unsigned short) hashUInt64(kmerIdx, hashShift)` for every +// k-mer (kmermatcher.cpp:202,213,223). That is a *pure function of the k-mer +// index*, so two occurrences of the same k-mer always produce the same score and +// therefore always land in the same partition. Grouping only ever compares equal +// k-mers, so a partition can be grouped in isolation and the result is identical +// to grouping the whole database at once. This is what makes the partitioning +// lossless rather than an approximation -- unlike sequence-space sharding, which +// co-locates near-duplicates only about a third of the time at 90% identity. +class KmerPartitioner { +public: + // partitionCount must be a power of two and at most 65536, since the score + // it partitions is 16 bits wide. + explicit KmerPartitioner(unsigned int partitionCount); + + unsigned int getPartitionCount() const { return partitionCount; } + + // Partition of a k-mer, from the 16-bit hash kmermatcher already computed. + // Taking the high bits keeps whole contiguous runs of the score space + // together, which is the same space the stock split ranges carve up, so a + // partition is directly comparable to a stock split. + unsigned int partitionOf(unsigned short score) const { + return static_cast(score) >> shift; + } + +private: + unsigned int partitionCount; + unsigned int shift; +}; + +// One k-mer occurrence as stored in a bucket file. +// +// 24 bytes, packed. Two fields are here specifically to delete arrays that stock +// kmermatcher sizes by *key space* rather than by entry count, and which +// therefore cannot exist at 1e12: +// - seqLen removes `seqkey_to_len[dbKeySize]` (kmermatcher.cpp:1236), +// - id is carried explicitly so nothing needs a dense key-indexed side table. +// `countTable` (:1256) and `repSequence` (:1357) fall the same way. +struct __attribute__((__packed__)) KmerRecord { + uint64_t kmer; + // 48 bits is 2.8e14 keys, comfortably past the 1e12 target, and saves 2 bytes + // per record against a full 64-bit key -- 40 TB at 1e12 with 21 k-mers each. + uint8_t idBytes[6]; + uint16_t pos; + uint16_t seqLen; + // Residues flanking the k-mer, as kmermatcher's --include-adjacency 1 keeps. + uint8_t adjacent[6]; + + uint64_t getId() const { + uint64_t id = 0; + for (int i = 5; i >= 0; i--) { + id = (id << 8) | idBytes[i]; + } + return id; + } + + void setId(uint64_t id) { + for (int i = 0; i < 6; i++) { + idBytes[i] = static_cast(id & 0xFF); + id >>= 8; + } + } + + static const uint64_t MAX_ID = (static_cast(1) << 48) - 1; +}; + +// Converts a bucket record into the KmerPosition that assignGroup consumes, and +// back. Templated rather than including kmermatcher.h so this header stays free +// of the whole DBReader/Parameters chain; the caller instantiates it with the +// KmerPosition variant it uses. +// +// The variant must be one with IncludeSeqLen = true. That is the point of the +// record carrying seqLen: with IncludeSeqLen = false, KmerPosition::getSeqLen() +// reads the *static* `SeqLenData::seqkey_to_len` array, which is sized +// by key space (kmermatcher.cpp:1236) and so cannot exist at 1e12. Reading the +// length out of the record is what deletes it. +template +void kmerRecordToPosition(const KmerRecord &record, KmerPositionT &out) { + out.kmer = record.kmer; + out.id = static_cast(record.getId()); + out.pos = static_cast(record.pos); + out.sl.setSeqLen(static_cast(record.seqLen)); + for (int i = 0; i < 6; i++) { + out.setAdjacentSeq(i, record.adjacent[i]); + } +} + +template +void kmerPositionToRecord(KmerPositionT &in, KmerRecord &out) { + out.kmer = in.kmer; + out.setId(static_cast(in.id)); + out.pos = static_cast(in.pos); + out.seqLen = static_cast(in.getSeqLen()); + for (int i = 0; i < 6; i++) { + out.adjacent[i] = in.getAdjacentSeq(i); + } +} + +// Appends k-mer records into per-partition bucket files. +// +// Each writer owns one shard id and writes /p/.kmers, so +// concurrent writers -- threads within a worker, and workers across nodes -- +// never touch the same file and need no locking at all. Bucket files are laid +// out one directory per partition because the reduce stage reads exactly one +// partition, and because a flat directory would hold partitions x shards entries. +class KmerBucketWriter { +public: + // bufferBudgetBytes is split evenly across partitions, so memory is bounded + // regardless of partition count. Records accumulate per partition and are + // flushed in large contiguous appends rather than one write per k-mer. + KmerBucketWriter(const std::string &dir, unsigned int partitionCount, + const std::string &shardId, size_t bufferBudgetBytes = 32 * 1024 * 1024); + ~KmerBucketWriter(); + + void append(unsigned int partition, const KmerRecord &record); + // Flushes and closes every open bucket. Called by the destructor, but call it + // explicitly to see write errors before the object goes away. + void close(); + + uint64_t getRecordCount() const { return recordCount; } + + // Creates the per-partition directories. Safe to call from many workers. + static void createLayout(const std::string &dir, unsigned int partitionCount); + static std::string partitionDir(const std::string &dir, unsigned int partition); + +private: + KmerBucketWriter(const KmerBucketWriter &); + KmerBucketWriter &operator=(const KmerBucketWriter &); + + void flush(unsigned int partition); + + std::string dir; + std::string shardId; + unsigned int partitionCount; + size_t recordsPerBuffer; + std::vector > buffers; + std::vector files; + uint64_t recordCount; +}; + +// Reads every shard of one partition back. +class KmerBucketReader { +public: + // Total records across all shards of the partition, so the reduce stage can + // size its array in one allocation before reading. + static uint64_t countRecords(const std::string &dir, unsigned int partition); + + // Appends every record of the partition to out. + static void readPartition(const std::string &dir, unsigned int partition, + std::vector &out); + + static std::vector shardFiles(const std::string &dir, unsigned int partition); +}; + +#endif diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index faca7f86a..42e3f1013 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -18,10 +18,14 @@ set(TESTS TestKmerScore.cpp TestKwayMerge.cpp TestMultipleAlignment.cpp + TestParallelCoordination.cpp TestProfileAlignment.cpp TestPSSM.cpp TestPSSMPrune.cpp TestDBReaderZstd.cpp + TestDenseIndex.cpp + TestKmerPartition.cpp + TestLengthRankedPlan.cpp TestReduceMatrix.cpp TestScoreMatrixSerialization.cpp TestSequenceIndex.cpp diff --git a/src/test/TestDenseIndex.cpp b/src/test/TestDenseIndex.cpp new file mode 100644 index 000000000..53a53f186 --- /dev/null +++ b/src/test/TestDenseIndex.cpp @@ -0,0 +1,176 @@ +// Tests for the fixed-width companion index used to open a key range of a +// database without materialising the whole text index. +// +// The property under test is the one the distributed stages depend on: a reader +// built from a key range must return byte-identical sequences to a reader over +// the whole database, while only ever reading that range's slice of the index. + +#include "DBReader.h" +#include "DBWriter.h" +#include "DenseIndex.h" +#include "Parameters.h" + +#include +#include +#include +#include + +#include + +const char* binary_name = "test_denseindex"; + +static int failures = 0; + +static void check(bool condition, const std::string &what) { + if (condition == false) { + fprintf(stderr, "FAIL: %s\n", what.c_str()); + failures++; + } else { + fprintf(stdout, "ok: %s\n", what.c_str()); + } +} + +static std::string makeTempDir() { + char tmpl[] = "/tmp/mmseqs_denseidx_testXXXXXX"; + char *dir = mkdtemp(tmpl); + if (dir == NULL) { + perror("mkdtemp"); + exit(EXIT_FAILURE); + } + return std::string(dir); +} + +static void removeTempDir(const std::string &dir) { + std::string cmd = "rm -rf '" + dir + "'"; + if (system(cmd.c_str()) != 0) { + fprintf(stderr, "warning: could not clean up %s\n", dir.c_str()); + } +} + +// Deliberately varies sequence length with the key so that a wrong offset shows +// up as a length mismatch as well as a content mismatch. +static std::string makeSequence(size_t key) { + const char alphabet[] = "ACDEFGHIKLMNPQRSTVWY"; + const size_t length = 5 + (key % 37); + std::string sequence; + for (size_t i = 0; i < length; i++) { + sequence.push_back(alphabet[(key + i) % 20]); + } + return sequence; +} + +static void writeDatabase(const std::string &dbName, size_t count) { + DBWriter writer(dbName.c_str(), (dbName + ".index").c_str(), 1, 0, + Parameters::DBTYPE_AMINO_ACIDS); + writer.open(); + for (size_t key = 0; key < count; key++) { + std::string sequence = makeSequence(key); + sequence.push_back('\n'); + writer.writeData(sequence.c_str(), sequence.size(), static_cast(key), 0); + } + writer.close(); +} + +// Opens [keyFrom, keyTo) through the companion index and hands the resulting +// array to DBReader's external-index constructor -- exactly the call sequence the +// distributed stages use. +static DBReader *openRange(const std::string &dbName, DBKeyType keyFrom, DBKeyType keyTo, + DBReader::Index **indexOut) { + DenseIndex::Info info; + DBReader::Index *index = DenseIndex::loadRange(dbName, keyFrom, keyTo, &info); + DBReader *reader = new DBReader( + index, info.entryCount, info.dataSize, + static_cast(keyTo == keyFrom ? keyFrom : keyTo - 1), + Parameters::DBTYPE_AMINO_ACIDS, info.maxSeqLen, 1); + reader->setDataFile(dbName.c_str()); + reader->open(DBReader::NOSORT); + *indexOut = index; + return reader; +} + +int main(int, const char**) { + const std::string dir = makeTempDir(); + const std::string dbName = dir + "/seqdb"; + const size_t count = 5000; + + writeDatabase(dbName, count); + DenseIndex::build(dbName); + + check(DenseIndex::exists(dbName), "companion index file is created"); + + DenseIndex::Info whole = DenseIndex::readInfo(dbName); + check(whole.entryCount == count, "companion index covers every entry"); + check(whole.firstKey == 0, "companion index records the first key"); + + // Reference: the database read the normal way. + DBReader full(dbName.c_str(), (dbName + ".index").c_str(), 1, + DBReader::USE_DATA | DBReader::USE_INDEX); + full.open(DBReader::NOSORT); + check(full.getSize() == count, "reference reader sees every entry"); + + // A range in the middle, a range at the start, and a range running to the end + // -- the three cases where an off-by-one in the row arithmetic would show up. + const DBKeyType ranges[][2] = { {1000, 1500}, {0, 64}, {4900, 5000}, {2500, 2501} }; + for (size_t r = 0; r < sizeof(ranges) / sizeof(ranges[0]); r++) { + const DBKeyType from = ranges[r][0]; + const DBKeyType to = ranges[r][1]; + + DBReader::Index *index = NULL; + DBReader *ranged = openRange(dbName, from, to, &index); + + bool sizeOk = ranged->getSize() == static_cast(to - from); + bool contentOk = true; + bool keysOk = true; + for (size_t i = 0; i < ranged->getSize(); i++) { + const DBKeyType expectedKey = static_cast(from + i); + if (ranged->getDbKey(i) != expectedKey) { + keysOk = false; + break; + } + const size_t fullId = full.getId(expectedKey); + if (ranged->getSeqLen(i) != full.getSeqLen(fullId)) { + contentOk = false; + break; + } + if (memcmp(ranged->getData(i, 0), full.getData(fullId, 0), + full.getEntryLen(fullId)) != 0) { + contentOk = false; + break; + } + } + + const std::string label = "[" + std::to_string(from) + ", " + std::to_string(to) + ")"; + check(sizeOk, "range " + label + " has the expected entry count"); + check(keysOk, "range " + label + " reports the expected keys"); + check(contentOk, "range " + label + " returns byte-identical sequences"); + + ranged->close(); + delete ranged; + delete[] index; + } + + // The whole database as one range must agree with the reference reader + // entry for entry, which also covers the data-size and max-length totals. + { + DBReader::Index *index = NULL; + DBReader *ranged = openRange(dbName, 0, static_cast(count), &index); + bool allOk = ranged->getSize() == count; + for (size_t i = 0; allOk && i < count; i++) { + allOk = ranged->getSeqLen(i) == full.getSeqLen(full.getId(static_cast(i))); + } + check(allOk, "full-database range matches the reference reader"); + ranged->close(); + delete ranged; + delete[] index; + } + + full.close(); + removeTempDir(dir); + + if (failures > 0) { + fprintf(stderr, "\n%d check(s) failed\n", failures); + return EXIT_FAILURE; + } + fprintf(stdout, "\nall checks passed\n"); + return EXIT_SUCCESS; +} diff --git a/src/test/TestKmerPartition.cpp b/src/test/TestKmerPartition.cpp new file mode 100644 index 000000000..f2b886342 --- /dev/null +++ b/src/test/TestKmerPartition.cpp @@ -0,0 +1,281 @@ +// Tests for k-mer space partitioning, the basis of the distributed kmermatcher. +// +// The property everything rests on is that partitioning k-mer space is +// *lossless*: every occurrence of a given k-mer lands in exactly one partition, +// so a partition can be grouped in isolation and the union of the partitions is +// exactly the input. If that ever fails, candidate pairs go missing silently and +// the clustering is quietly wrong rather than visibly broken -- so it is checked +// here directly, on a corpus with heavy k-mer repetition. + +#include "KmerPartition.h" +#include "kmermatcher.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +const char* binary_name = "test_kmerpartition"; + +static int failures = 0; + +static void check(bool condition, const std::string &what) { + if (condition == false) { + fprintf(stderr, "FAIL: %s\n", what.c_str()); + failures++; + } else { + fprintf(stdout, "ok: %s\n", what.c_str()); + } +} + +static std::string makeTempDir() { + char tmpl[] = "/tmp/mmseqs_kmerpart_testXXXXXX"; + char *dir = mkdtemp(tmpl); + if (dir == NULL) { + perror("mkdtemp"); + exit(EXIT_FAILURE); + } + return std::string(dir); +} + +static void removeTempDir(const std::string &dir) { + std::string cmd = "rm -rf '" + dir + "'"; + if (system(cmd.c_str()) != 0) { + fprintf(stderr, "could not remove %s\n", dir.c_str()); + } +} + +// Stands in for kmermatcher's `(unsigned short) hashUInt64(kmerIdx, hashShift)`. +// The only property the partitioner relies on is that it is a pure function of +// the k-mer, which this reproduces. +static unsigned short scoreOf(uint64_t kmer) { + uint64_t h = kmer * 0x9E3779B97F4A7C15ULL; + h ^= h >> 29; + h *= 0xBF58476D1CE4E5B9ULL; + h ^= h >> 32; + return static_cast(h & 0xFFFF); +} + +static void testRecordLayout() { + check(sizeof(KmerRecord) == 24, "k-mer record is 24 packed bytes"); + + KmerRecord record; + record.kmer = 0xDEADBEEFCAFEF00DULL; + record.pos = 4242; + record.seqLen = 65535; + for (int i = 0; i < 6; i++) { + record.adjacent[i] = static_cast(i + 1); + } + + // Boundary ids: the 48-bit field must survive its extremes intact, since a + // silent truncation here would corrupt sequence identity, not just size. + const uint64_t ids[] = {0, 1, 255, 256, 1000000000000ULL, KmerRecord::MAX_ID}; + bool allRoundTrip = true; + for (size_t i = 0; i < sizeof(ids) / sizeof(ids[0]); i++) { + record.setId(ids[i]); + allRoundTrip = allRoundTrip && record.getId() == ids[i]; + } + check(allRoundTrip, "48-bit id round-trips at 0, 1, 255, 256, 1e12 and 2^48-1"); + check(KmerRecord::MAX_ID > 1000000000000ULL, "id field spans past the 1e12 target"); + + record.setId(1000000000000ULL); + check(record.kmer == 0xDEADBEEFCAFEF00DULL && record.pos == 4242 && record.seqLen == 65535 && + record.adjacent[5] == 6, + "setting the id leaves the neighbouring packed fields untouched"); +} + +static void testPartitioner() { + KmerPartitioner partitioner(8192); + check(partitioner.getPartitionCount() == 8192, "partitioner reports its partition count"); + + // The plan sizes P = 8192 over the 16-bit hash space, i.e. 8 hash values per + // partition. Verify that is exactly what happens, and that the whole space + // maps inside range. + std::set seen; + bool inRange = true; + std::map perPartition; + for (unsigned int score = 0; score <= 65535; score++) { + const unsigned int p = partitioner.partitionOf(static_cast(score)); + inRange = inRange && p < 8192; + seen.insert(p); + perPartition[p]++; + } + check(inRange, "every 16-bit score maps inside the partition range"); + check(seen.size() == 8192, "every partition is reachable"); + bool evenlySized = true; + for (std::map::const_iterator it = perPartition.begin(); + it != perPartition.end(); ++it) { + evenlySized = evenlySized && it->second == 8; + } + check(evenlySized, "each of 8192 partitions covers exactly 8 hash values"); + + KmerPartitioner single(1); + bool allZero = true; + for (unsigned int score = 0; score <= 65535; score += 97) { + allZero = allZero && single.partitionOf(static_cast(score)) == 0; + } + check(allZero, "a single partition collects the whole hash space"); +} + +// The load-bearing test: run a repetitive corpus through the writer, read it +// back partition by partition, and prove nothing was lost, duplicated or split. +static void testLosslessRoundTrip(const std::string &dir) { + const unsigned int partitionCount = 64; + const std::string bucketDir = dir + "/buckets"; + KmerBucketWriter::createLayout(bucketDir, partitionCount); + KmerPartitioner partitioner(partitionCount); + + // Few distinct k-mers over many sequences, so most k-mers recur -- which is + // the case where a partitioning bug would actually lose candidate pairs. + const uint64_t distinctKmers = 500; + std::map > expected; // kmer -> sequence ids + srand(20260728); + + // Three shards, standing in for three threads or three worker nodes. + const char *shardNames[] = {"w0", "w1", "w2"}; + std::vector writers; + for (int s = 0; s < 3; s++) { + writers.push_back(new KmerBucketWriter(bucketDir, partitionCount, shardNames[s], 4096)); + } + + uint64_t written = 0; + for (uint64_t seqId = 0; seqId < 20000; seqId++) { + const int shard = static_cast(seqId % 3); + for (int k = 0; k < 21; k++) { + const uint64_t kmer = static_cast(rand()) % distinctKmers; + KmerRecord record; + record.kmer = kmer; + record.setId(seqId); + record.pos = static_cast(k); + record.seqLen = static_cast(100 + (seqId % 400)); + for (int i = 0; i < 6; i++) { + record.adjacent[i] = static_cast((kmer + i) & 0xFF); + } + writers[shard]->append(partitioner.partitionOf(scoreOf(kmer)), record); + expected[kmer].push_back(seqId); + written++; + } + } + for (int s = 0; s < 3; s++) { + writers[s]->close(); + delete writers[s]; + } + + uint64_t counted = 0; + for (unsigned int p = 0; p < partitionCount; p++) { + counted += KmerBucketReader::countRecords(bucketDir, p); + } + check(counted == written, "every written record is counted back across partitions"); + + // Read each partition and check the k-mers it holds belong to it alone. + std::map kmerPartition; + std::map > recovered; + bool fieldsIntact = true; + bool partitionPure = true; + uint64_t readBack = 0; + for (unsigned int p = 0; p < partitionCount; p++) { + std::vector records; + KmerBucketReader::readPartition(bucketDir, p, records); + for (size_t i = 0; i < records.size(); i++) { + const KmerRecord &record = records[i]; + // Every occurrence of this k-mer must be in this partition and no other. + if (kmerPartition.count(record.kmer) == 0) { + kmerPartition[record.kmer] = p; + } else if (kmerPartition[record.kmer] != p) { + partitionPure = false; + } + if (partitioner.partitionOf(scoreOf(record.kmer)) != p) { + partitionPure = false; + } + const uint64_t seqId = record.getId(); + fieldsIntact = fieldsIntact && record.seqLen == 100 + (seqId % 400) && + record.adjacent[0] == static_cast(record.kmer & 0xFF); + recovered[record.kmer].push_back(seqId); + readBack++; + } + } + + check(readBack == written, "every record survives the write/read round trip"); + check(partitionPure, "each k-mer appears in exactly one partition"); + check(fieldsIntact, "id, seqLen and adjacency survive the round trip"); + + bool sameMultiset = recovered.size() == expected.size(); + for (std::map >::iterator it = recovered.begin(); + sameMultiset && it != recovered.end(); ++it) { + std::vector &got = it->second; + std::vector want = expected[it->first]; + std::sort(got.begin(), got.end()); + std::sort(want.begin(), want.end()); + sameMultiset = got == want; + } + check(sameMultiset, + "grouping partition by partition sees exactly the same k-mer/sequence pairs as the whole input"); + + std::vector empty; + KmerBucketReader::readPartition(bucketDir + "_missing", 0, empty); + check(empty.empty() && KmerBucketReader::countRecords(bucketDir + "_missing", 0) == 0, + "a partition nothing was written to reads back empty rather than failing"); +} + +// The reduce stage feeds records into assignGroup through KmerPosition, so the +// record has to carry everything that struct exposes. In particular seqLen must +// come back out of getSeqLen() without any global key-indexed table, which is the +// whole reason the field is in the record. +static void testKmerPositionConversion() { + typedef KmerPosition Position; + + KmerRecord record; + record.kmer = 0x0123456789ABCDEFULL; + record.setId(999999999999ULL); + record.pos = 517; + record.seqLen = 30000; + for (int i = 0; i < 6; i++) { + record.adjacent[i] = static_cast(i * 3 + 1); + } + + Position position; + kmerRecordToPosition(record, position); + + bool adjacencyKept = true; + for (int i = 0; i < 6; i++) { + adjacencyKept = adjacencyKept && position.getAdjacentSeq(i) == static_cast(i * 3 + 1); + } + check(position.kmer == record.kmer && static_cast(position.id) == 999999999999ULL && + position.pos == 517, + "record converts into the KmerPosition assignGroup consumes"); + check(position.getSeqLen() == 30000, + "sequence length comes from the record, needing no key-indexed length table"); + check(adjacencyKept, "adjacency survives conversion into KmerPosition"); + + KmerRecord back; + kmerPositionToRecord(position, back); + bool sameBytes = back.kmer == record.kmer && back.getId() == record.getId() && + back.pos == record.pos && back.seqLen == record.seqLen; + for (int i = 0; i < 6; i++) { + sameBytes = sameBytes && back.adjacent[i] == record.adjacent[i]; + } + check(sameBytes, "KmerPosition converts back into an identical record"); +} + +int main(int, char **) { + const std::string dir = makeTempDir(); + + testRecordLayout(); + testPartitioner(); + testKmerPositionConversion(); + testLosslessRoundTrip(dir); + + removeTempDir(dir); + + if (failures > 0) { + fprintf(stderr, "\n%d check(s) failed\n", failures); + return EXIT_FAILURE; + } + fprintf(stdout, "\nall checks passed\n"); + return EXIT_SUCCESS; +} diff --git a/src/test/TestLengthRankedPlan.cpp b/src/test/TestLengthRankedPlan.cpp new file mode 100644 index 000000000..d2e0ac833 --- /dev/null +++ b/src/test/TestLengthRankedPlan.cpp @@ -0,0 +1,281 @@ +// Tests for the placement math behind the distributed, length-ranked createdb. +// +// Pass 2 writes sequences straight to their final byte offsets with no merge and +// no communication between workers, so the plan is the only thing keeping those +// writes from colliding. The properties under test are therefore the ones that +// make the output a valid database at all: +// +// - keys tile [0, N) exactly: no key unassigned, none assigned twice; +// - keys are length-ranked: reading in key order yields non-increasing lengths; +// - data and header byte ranges tile their files exactly, so no worker can +// overwrite another's bytes and no gap is left uninitialised; +// - the plan depends only on the input, not on the order histograms arrive in. + +#include "LengthRankedPlan.h" + +#include +#include +#include +#include +#include + +#include + +const char* binary_name = "test_lengthrankedplan"; + +static int failures = 0; + +static void check(bool condition, const std::string &what) { + if (condition == false) { + fprintf(stderr, "FAIL: %s\n", what.c_str()); + failures++; + } else { + fprintf(stdout, "ok: %s\n", what.c_str()); + } +} + +static std::string makeTempDir() { + char tmpl[] = "/tmp/mmseqs_lrplan_testXXXXXX"; + char *dir = mkdtemp(tmpl); + if (dir == NULL) { + perror("mkdtemp"); + exit(EXIT_FAILURE); + } + return std::string(dir); +} + +static void removeTempDir(const std::string &dir) { + std::string cmd = "rm -rf '" + dir + "'"; + if (system(cmd.c_str()) != 0) { + fprintf(stderr, "could not remove %s\n", dir.c_str()); + } +} + +static ChunkHistogram makeHistogram(uint64_t chunkIdx, uint64_t fileIdx, + const std::vector &buckets) { + ChunkHistogram histogram; + histogram.chunkIdx = chunkIdx; + histogram.fileIdx = fileIdx; + histogram.buckets = buckets; + for (size_t i = 0; i < buckets.size(); i++) { + histogram.seqCount += buckets[i].count; + } + return histogram; +} + +static ChunkHistogram::Bucket bucket(uint64_t length, uint64_t count, uint64_t headerBytes) { + ChunkHistogram::Bucket b; + b.length = length; + b.count = count; + b.headerBytes = headerBytes; + return b; +} + +// A worked two-chunk example, checked entry by entry against hand-computed +// offsets. The randomised test below proves the invariants hold in general; this +// one pins down that the arithmetic itself is the intended arithmetic. +static void testWorkedExample() { + std::vector histograms; + histograms.push_back(makeHistogram(0, 0, {bucket(5, 2, 20), bucket(10, 1, 12)})); + histograms.push_back(makeHistogram(1, 0, {bucket(5, 3, 33), bucket(7, 1, 9)})); + + std::vector plans; + LengthRankedTotals totals = buildLengthRankedPlan(histograms, plans); + + check(totals.seqCount == 7, "worked example totals 7 sequences"); + // 1*(10+2) + 1*(7+2) + 5*(5+2) = 12 + 9 + 35 + check(totals.dataBytes == 56, "worked example totals 56 data bytes"); + check(totals.headerBytes == 74, "worked example totals 74 header bytes"); + check(totals.maxSeqLen == 10, "worked example reports the longest sequence"); + + check(plans.size() == 2, "worked example produces one plan per chunk"); + // Entries are stored ascending by length. + check(plans[0].entries.size() == 2 && plans[1].entries.size() == 2, + "worked example plans keep every bucket"); + + const ChunkPlan::Entry &c0len10 = plans[0].entries[1]; + check(c0len10.length == 10 && c0len10.keyStart == 0 && c0len10.dataOffset == 0 && c0len10.hdrOffset == 0, + "longest sequence takes key 0 at offset 0"); + + const ChunkPlan::Entry &c1len7 = plans[1].entries[1]; + check(c1len7.length == 7 && c1len7.keyStart == 1 && c1len7.dataOffset == 12 && c1len7.hdrOffset == 12, + "second-longest sequence follows the longest"); + + // Both chunks hold length 5; chunk 0 must come first because ties break by + // chunk index, which is input order. + const ChunkPlan::Entry &c0len5 = plans[0].entries[0]; + const ChunkPlan::Entry &c1len5 = plans[1].entries[0]; + check(c0len5.keyStart == 2 && c0len5.dataOffset == 21 && c0len5.hdrOffset == 21, + "lower chunk index wins a length tie"); + check(c1len5.keyStart == 4 && c1len5.dataOffset == 35 && c1len5.hdrOffset == 41, + "higher chunk index follows within the same length"); +} + +// Materialises every (key, length, dataOffset, headerBytes) run the plan implies +// and checks the runs tile all three address spaces exactly. +static void testTilingInvariants() { + srand(20260728); + + for (int trial = 0; trial < 20; trial++) { + const size_t chunkCount = 1 + (rand() % 12); + std::vector histograms; + for (size_t c = 0; c < chunkCount; c++) { + // Lengths are drawn from a small shared pool so chunks collide on + // lengths often, which is what exercises the tie-breaking. + std::vector buckets; + for (uint64_t length = 1; length <= 20; length++) { + if (rand() % 3 == 0) { + continue; + } + const uint64_t count = 1 + (rand() % 5); + buckets.push_back(bucket(length, count, count * (3 + (rand() % 7)))); + } + histograms.push_back(makeHistogram(c, c % 3, buckets)); + } + + std::vector reference = histograms; + std::vector plans; + LengthRankedTotals totals = buildLengthRankedPlan(histograms, plans); + + // Flatten the plan into runs ordered by key. + struct Run { + uint64_t keyStart, count, length, dataOffset, hdrOffset, hdrBytes; + }; + std::vector runs; + for (size_t i = 0; i < plans.size(); i++) { + for (size_t e = 0; e < plans[i].entries.size(); e++) { + const ChunkPlan::Entry &entry = plans[i].entries[e]; + uint64_t hdrBytes = 0; + for (size_t b = 0; b < histograms[i].buckets.size(); b++) { + if (histograms[i].buckets[b].length == entry.length) { + hdrBytes = histograms[i].buckets[b].headerBytes; + } + } + Run run = {entry.keyStart, entry.count, entry.length, + entry.dataOffset, entry.hdrOffset, hdrBytes}; + runs.push_back(run); + } + } + std::sort(runs.begin(), runs.end(), + [](const Run &a, const Run &b) { return a.keyStart < b.keyStart; }); + + uint64_t expectedKey = 0, expectedData = 0, expectedHdr = 0; + uint64_t previousLength = UINT64_MAX; + bool keysTile = true, dataTiles = true, hdrTiles = true, lengthRanked = true; + for (size_t r = 0; r < runs.size(); r++) { + keysTile = keysTile && runs[r].keyStart == expectedKey; + dataTiles = dataTiles && runs[r].dataOffset == expectedData; + hdrTiles = hdrTiles && runs[r].hdrOffset == expectedHdr; + lengthRanked = lengthRanked && runs[r].length <= previousLength; + + previousLength = runs[r].length; + expectedKey += runs[r].count; + expectedData += runs[r].count * (runs[r].length + 2); + expectedHdr += runs[r].hdrBytes; + } + + if (trial == 0) { + check(keysTile, "key ranges tile [0, N) with no gap or overlap"); + check(dataTiles, "data byte ranges tile the data file exactly"); + check(hdrTiles, "header byte ranges tile the header file exactly"); + check(lengthRanked, "sequences are ordered longest first in key order"); + check(expectedKey == totals.seqCount, "totals agree with the tiled key count"); + check(expectedData == totals.dataBytes, "totals agree with the tiled data size"); + check(expectedHdr == totals.headerBytes, "totals agree with the tiled header size"); + } else if (!keysTile || !dataTiles || !hdrTiles || !lengthRanked || + expectedKey != totals.seqCount || expectedData != totals.dataBytes || + expectedHdr != totals.headerBytes) { + check(false, "tiling invariants hold on randomised trial " + std::to_string(trial)); + return; + } + + // Feeding the histograms in a different order must not change the plan: + // workers finish in arbitrary order, so the planner must not depend on it. + std::vector shuffled = reference; + std::reverse(shuffled.begin(), shuffled.end()); + std::vector shuffledPlans; + buildLengthRankedPlan(shuffled, shuffledPlans); + bool identical = shuffledPlans.size() == plans.size(); + for (size_t i = 0; identical && i < plans.size(); i++) { + identical = shuffledPlans[i].chunkIdx == plans[i].chunkIdx && + shuffledPlans[i].entries.size() == plans[i].entries.size(); + for (size_t e = 0; identical && e < plans[i].entries.size(); e++) { + identical = shuffledPlans[i].entries[e].keyStart == plans[i].entries[e].keyStart && + shuffledPlans[i].entries[e].dataOffset == plans[i].entries[e].dataOffset && + shuffledPlans[i].entries[e].hdrOffset == plans[i].entries[e].hdrOffset; + } + } + if (trial == 0) { + check(identical, "plan is independent of the order histograms are collected in"); + } else if (!identical) { + check(false, "plan stayed order-independent on randomised trial " + std::to_string(trial)); + return; + } + } + check(true, "tiling invariants hold across 20 randomised chunk layouts"); +} + +static void testRoundTrip(const std::string &dir) { + ChunkHistogram histogram = makeHistogram(7, 3, {bucket(4, 9, 40), bucket(11, 2, 18)}); + histogram.nuclVotes = 5; + histogram.sampleCount = 11; + const std::string histPath = dir + "/chunk7.hist"; + histogram.write(histPath); + ChunkHistogram loaded = ChunkHistogram::read(histPath); + + bool same = loaded.chunkIdx == 7 && loaded.fileIdx == 3 && loaded.seqCount == 11 && + loaded.nuclVotes == 5 && loaded.sampleCount == 11 && + loaded.buckets.size() == 2; + for (size_t i = 0; same && i < loaded.buckets.size(); i++) { + same = loaded.buckets[i].length == histogram.buckets[i].length && + loaded.buckets[i].count == histogram.buckets[i].count && + loaded.buckets[i].headerBytes == histogram.buckets[i].headerBytes; + } + check(same, "chunk histogram survives a write/read round trip"); + + std::vector histograms(1, histogram); + std::vector plans; + buildLengthRankedPlan(histograms, plans); + const std::string planPath = dir + "/chunk7.plan"; + plans[0].write(planPath); + ChunkPlan loadedPlan = ChunkPlan::read(planPath); + + bool planSame = loadedPlan.chunkIdx == plans[0].chunkIdx && + loadedPlan.fileIdx == plans[0].fileIdx && + loadedPlan.entries.size() == plans[0].entries.size(); + for (size_t i = 0; planSame && i < loadedPlan.entries.size(); i++) { + planSame = loadedPlan.entries[i].length == plans[0].entries[i].length && + loadedPlan.entries[i].count == plans[0].entries[i].count && + loadedPlan.entries[i].keyStart == plans[0].entries[i].keyStart && + loadedPlan.entries[i].dataOffset == plans[0].entries[i].dataOffset && + loadedPlan.entries[i].hdrOffset == plans[0].entries[i].hdrOffset; + } + check(planSame, "chunk plan survives a write/read round trip"); +} + +static void testEmptyInput() { + std::vector histograms; + histograms.push_back(makeHistogram(0, 0, {})); + std::vector plans; + LengthRankedTotals totals = buildLengthRankedPlan(histograms, plans); + check(totals.seqCount == 0 && totals.dataBytes == 0 && totals.headerBytes == 0, + "an empty chunk plans to an empty database"); +} + +int main(int, char **) { + const std::string dir = makeTempDir(); + + testWorkedExample(); + testTilingInvariants(); + testRoundTrip(dir); + testEmptyInput(); + + removeTempDir(dir); + + if (failures > 0) { + fprintf(stderr, "\n%d check(s) failed\n", failures); + return EXIT_FAILURE; + } + fprintf(stdout, "\nall checks passed\n"); + return EXIT_SUCCESS; +} diff --git a/src/test/TestParallelCoordination.cpp b/src/test/TestParallelCoordination.cpp new file mode 100644 index 000000000..713a17197 --- /dev/null +++ b/src/test/TestParallelCoordination.cpp @@ -0,0 +1,299 @@ +// Tests for the shared-filesystem coordination layer. +// +// The interesting failure modes only appear across *processes*: fcntl locks are +// owned per-process, so a thread-only test would pass even if the locking were +// completely absent. The concurrency tests therefore fork real children, which is +// also how the layer is used in production (one worker process per node). + +#include "ParallelCoordination.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +const char* binary_name = "test_parallelcoordination"; + +static int failures = 0; + +static void check(bool condition, const std::string &what) { + if (condition == false) { + fprintf(stderr, "FAIL: %s\n", what.c_str()); + failures++; + } else { + fprintf(stdout, "ok: %s\n", what.c_str()); + } +} + +static std::string makeTempDir() { + char tmpl[] = "/tmp/mmseqs_coord_testXXXXXX"; + char *dir = mkdtemp(tmpl); + if (dir == NULL) { + perror("mkdtemp"); + exit(EXIT_FAILURE); + } + return std::string(dir); +} + +static void removeTempDir(const std::string &dir) { + std::string cmd = "rm -rf '" + dir + "'"; + if (system(cmd.c_str()) != 0) { + fprintf(stderr, "warning: could not clean up %s\n", dir.c_str()); + } +} + +// Children report the values they observed by appending them to their own file, +// which avoids needing any shared memory between parent and children. +static void writeValues(const std::string &path, const std::vector &values) { + FILE *file = fopen(path.c_str(), "w"); + if (file == NULL) { + perror(path.c_str()); + exit(EXIT_FAILURE); + } + for (size_t i = 0; i < values.size(); i++) { + fprintf(file, "%lld\n", static_cast(values[i])); + } + fclose(file); +} + +static std::vector readValues(const std::string &path) { + std::vector values; + FILE *file = fopen(path.c_str(), "r"); + if (file == NULL) { + return values; + } + long long value; + while (fscanf(file, "%lld", &value) == 1) { + values.push_back(static_cast(value)); + } + fclose(file); + return values; +} + +// Every fetchAdd must return a distinct value even when several processes race, +// because worker identity is assigned this way and duplicate ids would make two +// workers write to the same output shard. +static void testCounterAcrossProcesses(const std::string &dir) { + const std::string counterPath = dir + "/worker_id"; + const int childCount = 8; + const int perChild = 25; + + for (int child = 0; child < childCount; child++) { + pid_t pid = fork(); + if (pid == 0) { + SharedCounter counter(counterPath); + std::vector mine; + for (int i = 0; i < perChild; i++) { + mine.push_back(counter.fetchAdd(1)); + } + writeValues(dir + "/counter_child_" + std::to_string(child), mine); + _exit(EXIT_SUCCESS); + } + } + for (int child = 0; child < childCount; child++) { + int status = 0; + wait(&status); + } + + std::set seen; + for (int child = 0; child < childCount; child++) { + std::vector values = readValues(dir + "/counter_child_" + std::to_string(child)); + for (size_t i = 0; i < values.size(); i++) { + seen.insert(values[i]); + } + } + + SharedCounter counter(counterPath); + check(counter.get() == childCount * perChild, "counter total is exact under process contention"); + check(seen.size() == static_cast(childCount * perChild), + "every fetchAdd returned a distinct value"); + check(*seen.begin() == 0 && *seen.rbegin() == childCount * perChild - 1, + "fetchAdd values form a dense range"); +} + +// Threads inside one process share the fcntl lock (it is per-process), so this +// exercises the process-local mutex that FileLock adds on top. +static void testCounterAcrossThreads(const std::string &dir) { + const std::string counterPath = dir + "/thread_counter"; + SharedCounter counter(counterPath); + + const int threadCount = 8; + const int perThread = 25; + std::vector > perThreadValues(threadCount); + std::vector threads; + for (int t = 0; t < threadCount; t++) { + threads.push_back(std::thread([&counter, &perThreadValues, t, perThread]() { + for (int i = 0; i < perThread; i++) { + perThreadValues[t].push_back(counter.fetchAdd(1)); + } + })); + } + for (size_t t = 0; t < threads.size(); t++) { + threads[t].join(); + } + + std::set seen; + for (int t = 0; t < threadCount; t++) { + for (size_t i = 0; i < perThreadValues[t].size(); i++) { + seen.insert(perThreadValues[t][i]); + } + } + check(seen.size() == static_cast(threadCount * perThread), + "every fetchAdd returned a distinct value across threads"); +} + +static void testQueueDrainsExactlyOnce(const std::string &dir) { + const std::string queuePath = dir + "/single_queue"; + const int64_t itemCount = 64; + WorkQueue queue(queuePath, itemCount); + + std::set claimed; + int64_t index; + while ((index = queue.claim(1)) >= 0) { + claimed.insert(index); + queue.complete(index, 1); + } + + check(claimed.size() == static_cast(itemCount), "single worker claims every item once"); + check(queue.allDone(), "queue reports all done after draining"); + check(queue.getDoneCount() == itemCount, "done count matches item count"); +} + +static void testQueueConcurrentClaims(const std::string &dir) { + const std::string queuePath = dir + "/concurrent_queue"; + const int64_t itemCount = 200; + const int childCount = 8; + + // Created up front so children do not race on initialisation semantics that + // the resume test covers separately. + { + WorkQueue queue(queuePath, itemCount); + check(queue.getDoneCount() == 0, "fresh queue starts empty"); + } + + for (int child = 0; child < childCount; child++) { + pid_t pid = fork(); + if (pid == 0) { + WorkQueue queue(queuePath, itemCount); + std::vector mine; + int64_t index; + while ((index = queue.claim(child + 1)) >= 0) { + mine.push_back(index); + queue.complete(index, child + 1); + } + writeValues(dir + "/queue_child_" + std::to_string(child), mine); + _exit(EXIT_SUCCESS); + } + } + for (int child = 0; child < childCount; child++) { + int status = 0; + wait(&status); + } + + std::vector all; + for (int child = 0; child < childCount; child++) { + std::vector values = readValues(dir + "/queue_child_" + std::to_string(child)); + all.insert(all.end(), values.begin(), values.end()); + } + std::set unique(all.begin(), all.end()); + + check(all.size() == static_cast(itemCount), "no item was handed out twice"); + check(unique.size() == static_cast(itemCount), "every item was handed out"); + + WorkQueue queue(queuePath, itemCount); + check(queue.allDone(), "queue is drained after concurrent workers finish"); +} + +// The property that makes a stage survive a 24 h walltime: a worker that dies +// holding a claim must not strand its item forever. +static void testLeaseExpiryRecovers(const std::string &dir) { + const std::string queuePath = dir + "/lease_queue"; + WorkQueue queue(queuePath, 4); + + int64_t first = queue.claim(1, 1); + check(first == 0, "first claim takes the lowest item"); + + int64_t second = queue.claim(2, 60); + check(second == 1, "a live claim is not stolen by another worker"); + + sleep(2); + + int64_t reclaimed = queue.claim(3, 60); + check(reclaimed == 0, "an expired claim is handed to another worker"); + + // The original holder must not be able to mark it done any more -- but if it + // does, the item stays done exactly once rather than double-counting. + queue.complete(0, 3); + queue.complete(0, 1); + check(queue.getDoneCount() == 1, "completing twice counts once"); +} + +static void testReleaseRequeues(const std::string &dir) { + const std::string queuePath = dir + "/release_queue"; + WorkQueue queue(queuePath, 4); + + int64_t index = queue.claim(1, 600); + check(index == 0, "claimed the first item"); + queue.release(index, 1); + + int64_t again = queue.claim(2, 600); + check(again == 0, "released item is immediately re-claimable"); +} + +// Reopening a queue must preserve progress; this is what lets a stage continue in +// a fresh Slurm job without redoing finished work. +static void testResumeKeepsProgress(const std::string &dir) { + const std::string queuePath = dir + "/resume_queue"; + const int64_t itemCount = 16; + + { + WorkQueue queue(queuePath, itemCount); + for (int i = 0; i < 8; i++) { + int64_t index = queue.claim(1); + queue.complete(index, 1); + } + check(queue.getDoneCount() == 8, "half the queue is done before restart"); + } + + { + WorkQueue queue(queuePath, itemCount); + check(queue.getDoneCount() == 8, "reopened queue remembers finished work"); + + std::set claimed; + int64_t index; + while ((index = queue.claim(2)) >= 0) { + claimed.insert(index); + queue.complete(index, 2); + } + check(claimed.size() == 8, "restart only redoes the unfinished items"); + check(queue.allDone(), "queue completes after restart"); + } +} + +int main(int, const char**) { + std::string dir = makeTempDir(); + + testCounterAcrossProcesses(dir); + testCounterAcrossThreads(dir); + testQueueDrainsExactlyOnce(dir); + testQueueConcurrentClaims(dir); + testLeaseExpiryRecovers(dir); + testReleaseRequeues(dir); + testResumeKeepsProgress(dir); + + removeTempDir(dir); + + if (failures > 0) { + fprintf(stderr, "\n%d check(s) failed\n", failures); + return EXIT_FAILURE; + } + fprintf(stdout, "\nall checks passed\n"); + return EXIT_SUCCESS; +} diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index df9ee3b14..f36c076db 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -14,6 +14,7 @@ set(util_source_files util/convertmsa.cpp util/convertprofiledb.cpp util/createdb.cpp + util/createdbparallel.cpp util/dbtype.cpp util/db2tar.cpp util/indexdb.cpp diff --git a/src/util/createdbparallel.cpp b/src/util/createdbparallel.cpp new file mode 100644 index 000000000..e62fdc8f8 --- /dev/null +++ b/src/util/createdbparallel.cpp @@ -0,0 +1,592 @@ +/* + * createdbparallel + * + * Builds a sequence database from FASTA input with many independent worker + * processes that coordinate only through a shared directory -- no MPI, no rank + * argument, and no direct node-to-node communication. Every worker runs the same + * command line; identity comes from a counter file. + * + * Two things make this different from stock createdb, and both exist to survive + * a trillion sequences: + * + * - Keys are *length-ranked and dense*: key 0 is the longest sequence and key + * == global order rank. That is what lets DenseIndex address an entry by key + * without a resident index, and what makes the distributed greedy in the + * reduce stage exact (order within a component is just sorted keys). + * - Nothing is merged. A first pass histograms sequence lengths per input byte + * range; those histograms alone determine every byte offset in the finished + * database (see LengthRankedPlan.h), so the second pass writes each sequence + * straight to its final position. Stock createdb's GPU mode achieves the same + * ordering, but only by sorting and then merging every shard through one + * process holding an entry per sequence in RAM. + * + * Memory per worker is bounded by --chunk-size times the thread count, not by + * the size of the input, which is the property the whole design rests on. + */ +#include "Command.h" +#include "Debug.h" +#include "DBReader.h" +#include "DBWriter.h" +#include "DenseIndex.h" +#include "FastSort.h" +#include "FileUtil.h" +#include "KSeqWrapper.h" +#include "LengthRankedPlan.h" +#include "ParallelCoordination.h" +#include "Parameters.h" +#include "Util.h" + +#include +#include +#include +#include +#include + +#include +#include + +#ifdef OPENMP +#include +#endif + +namespace { + +// One work item: the records whose '>' falls in [begin, end) of one input file. +struct Chunk { + size_t fileIdx; + size_t begin; + size_t end; +}; + +std::string chunkHistPath(const std::string &coordDir, size_t chunkIdx) { + return coordDir + "/chunk." + SSTR(chunkIdx) + ".hist"; +} + +std::string chunkPlanPath(const std::string &coordDir, size_t chunkIdx) { + return coordDir + "/chunk." + SSTR(chunkIdx) + ".plan"; +} + +void writeAt(int fd, const void *data, size_t length, size_t offset, const char *what) { + const char *cursor = static_cast(data); + size_t done = 0; + while (done < length) { + const ssize_t written = pwrite(fd, cursor + done, length - done, static_cast(offset + done)); + if (written <= 0) { + if (errno == EINTR) { + continue; + } + Debug(Debug::ERROR) << "Cannot write " << what << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + done += static_cast(written); + } +} + +int openForWrite(const std::string &path) { + const int fd = open(path.c_str(), O_WRONLY); + if (fd < 0) { + Debug(Debug::ERROR) << "Cannot open " << path << " for writing: " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + return fd; +} + +// Offset of the first record that starts at or after `from`. +// +// A record starts at a '>' that is either the first byte of the file or directly +// preceded by a newline. Resolving boundaries this way is what makes chunks +// independent: a record is owned by exactly the chunk its '>' falls in, so no +// record is split between chunks or claimed by two of them. +size_t findRecordStart(int fd, size_t from, size_t fileSize) { + if (from == 0) { + return 0; + } + if (from >= fileSize) { + return fileSize; + } + + const size_t windowSize = 65536; + std::vector window(windowSize); + // Start one byte early so a "\n>" straddling the requested offset is seen. + size_t pos = from - 1; + while (pos < fileSize) { + const size_t want = std::min(windowSize, fileSize - pos); + ssize_t got = pread(fd, window.data(), want, static_cast(pos)); + if (got <= 0) { + if (errno == EINTR) { + continue; + } + Debug(Debug::ERROR) << "Cannot read input file: " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + for (ssize_t i = 0; i + 1 < got; i++) { + if (window[i] == '\n' && window[i + 1] == '>') { + return pos + static_cast(i) + 1; + } + } + if (static_cast(got) < want) { + continue; + } + // Overlap by one byte so a pair split across windows is not missed. + pos += static_cast(got) - 1; + } + return fileSize; +} + +// Splits every input file into chunk-size pieces aligned to record boundaries. +// Computed identically and independently by every worker, so the chunk numbering +// -- and therefore the key assignment that breaks length ties by chunk index -- +// never depends on who is running. +std::vector planChunks(const std::vector &filenames, size_t chunkSize) { + std::vector chunks; + for (size_t fileIdx = 0; fileIdx < filenames.size(); fileIdx++) { + const size_t fileSize = FileUtil::getFileSize(filenames[fileIdx]); + if (fileSize == 0) { + continue; + } + const int fd = open(filenames[fileIdx].c_str(), O_RDONLY); + if (fd < 0) { + Debug(Debug::ERROR) << "Cannot open " << filenames[fileIdx] << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + size_t begin = 0; + while (begin < fileSize) { + const size_t nominalEnd = std::min(begin + chunkSize, fileSize); + const size_t end = findRecordStart(fd, nominalEnd, fileSize); + if (end > begin) { + Chunk chunk = {fileIdx, begin, end}; + chunks.push_back(chunk); + } + if (end <= begin) { + break; + } + begin = end; + } + close(fd); + } + return chunks; +} + +// Reads a chunk's bytes so they can be handed to the FASTA parser in one piece. +void readChunk(const std::string &filename, const Chunk &chunk, std::vector &buffer) { + const int fd = open(filename.c_str(), O_RDONLY); + if (fd < 0) { + Debug(Debug::ERROR) << "Cannot open " << filename << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + const size_t length = chunk.end - chunk.begin; + buffer.resize(length); + size_t done = 0; + while (done < length) { + const ssize_t got = pread(fd, buffer.data() + done, length - done, + static_cast(chunk.begin + done)); + if (got <= 0) { + if (errno == EINTR) { + continue; + } + Debug(Debug::ERROR) << "Cannot read " << filename << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + done += static_cast(got); + } + close(fd); +} + +// Builds the header exactly as createdb does, so a database built either way +// holds byte-identical header entries. +void buildHeader(const KSeqWrapper::KSeqEntry &entry, std::string &header) { + header.clear(); + header.append(entry.name.s, entry.name.l); + if (entry.comment.l > 0) { + header.append(" ", 1); + header.append(entry.comment.s, entry.comment.l); + } + header.push_back('\n'); +} + +// Pass 1: report what the chunk contains without writing any sequence data. +ChunkHistogram scanChunk(const std::string &filename, const Chunk &chunk, size_t chunkIdx) { + std::vector buffer; + readChunk(filename, chunk, buffer); + + // Counted into parallel arrays kept ascending by length. The number of + // distinct lengths is small (a few thousand for proteins) regardless of how + // big the chunk is, which is what keeps a histogram tens of KB rather than + // proportional to the sequence count. + std::vector lengthOf; + std::vector countOf; + std::vector headerBytesOf; + + KSeqBuffer kseq(buffer.data(), buffer.size()); + std::string header; + uint64_t seqCount = 0; + uint64_t nuclVotes = 0; + uint64_t sampleCount = 0; + const size_t testForNucSequence = 100; + + while (kseq.ReadEntry()) { + const KSeqWrapper::KSeqEntry &entry = kseq.entry; + if (entry.name.l == 0) { + Debug(Debug::ERROR) << "Invalid FASTA entry in " << filename << " at byte " + << chunk.begin << "\n"; + EXIT(EXIT_FAILURE); + } + buildHeader(entry, header); + + const uint64_t length = entry.sequence.l; + // Insertion sort into the ascending length list. Chunks hold few distinct + // lengths, so a linear probe from the end is cheaper than a hash map. + size_t slot = lengthOf.size(); + while (slot > 0 && lengthOf[slot - 1] > length) { + slot--; + } + if (slot == 0 || lengthOf[slot - 1] != length) { + lengthOf.insert(lengthOf.begin() + slot, length); + countOf.insert(countOf.begin() + slot, 0); + headerBytesOf.insert(headerBytesOf.begin() + slot, 0); + } else { + slot--; + } + countOf[slot]++; + // DBWriter appends a NUL after the header text. + headerBytesOf[slot] += header.length() + 1; + + if (sampleCount < testForNucSequence) { + size_t nucleotideLike = 0; + for (size_t i = 0; i < entry.sequence.l; i++) { + switch (toupper(entry.sequence.s[i])) { + case 'T': case 'A': case 'G': case 'C': case 'U': case 'N': + nucleotideLike++; + break; + } + } + if (entry.sequence.l > 0 && + static_cast(nucleotideLike) / static_cast(entry.sequence.l) > 0.9f) { + nuclVotes++; + } + sampleCount++; + } + seqCount++; + } + + ChunkHistogram histogram; + histogram.chunkIdx = chunkIdx; + histogram.fileIdx = chunk.fileIdx; + histogram.seqCount = seqCount; + histogram.nuclVotes = nuclVotes; + histogram.sampleCount = sampleCount; + histogram.buckets.resize(lengthOf.size()); + for (size_t i = 0; i < lengthOf.size(); i++) { + histogram.buckets[i].length = lengthOf[i]; + histogram.buckets[i].count = countOf[i]; + histogram.buckets[i].headerBytes = headerBytesOf[i]; + } + return histogram; +} + +// Everything one length bucket of one chunk writes. Buffered rather than written +// per sequence so each bucket becomes a single large, contiguous write instead of +// a scatter of ~250 byte writes over a parallel filesystem. +struct BucketOutput { + uint64_t keyStart; + uint64_t dataOffset; + uint64_t hdrOffset; + uint64_t written; + std::vector seqData; + std::vector hdrData; + std::vector seqIndex; + std::vector hdrIndex; +}; + +// Pass 2: rescan the chunk and write every sequence at its planned position. +void emitChunk(const std::string &filename, const Chunk &chunk, const ChunkPlan &plan, + int seqFd, int hdrFd, int seqIdxFd, int hdrIdxFd) { + std::vector buffer; + readChunk(filename, chunk, buffer); + + std::vector buckets(plan.entries.size()); + for (size_t i = 0; i < plan.entries.size(); i++) { + buckets[i].keyStart = plan.entries[i].keyStart; + buckets[i].dataOffset = plan.entries[i].dataOffset; + buckets[i].hdrOffset = plan.entries[i].hdrOffset; + buckets[i].written = 0; + } + + KSeqBuffer kseq(buffer.data(), buffer.size()); + std::string header; + while (kseq.ReadEntry()) { + const KSeqWrapper::KSeqEntry &entry = kseq.entry; + buildHeader(entry, header); + const uint64_t length = entry.sequence.l; + + // The plan lists one entry per length, ascending, so this is a binary + // search over a few thousand entries at most. + size_t lo = 0; + size_t hi = plan.entries.size(); + while (lo < hi) { + const size_t mid = lo + (hi - lo) / 2; + if (plan.entries[mid].length < length) { + lo = mid + 1; + } else { + hi = mid; + } + } + if (lo >= plan.entries.size() || plan.entries[lo].length != length) { + Debug(Debug::ERROR) << "Chunk " << plan.chunkIdx << " holds a sequence of length " + << length << " that pass 1 did not record. The input changed " + << "between the two passes.\n"; + EXIT(EXIT_FAILURE); + } + BucketOutput &bucket = buckets[lo]; + + DenseIndex::Entry seqEntry; + seqEntry.offset = bucket.dataOffset + bucket.seqData.size(); + seqEntry.length = static_cast(length + 2); + bucket.seqIndex.push_back(seqEntry); + bucket.seqData.insert(bucket.seqData.end(), entry.sequence.s, entry.sequence.s + length); + bucket.seqData.push_back('\n'); + bucket.seqData.push_back('\0'); + + DenseIndex::Entry hdrEntry; + hdrEntry.offset = bucket.hdrOffset + bucket.hdrData.size(); + hdrEntry.length = static_cast(header.length() + 1); + bucket.hdrIndex.push_back(hdrEntry); + bucket.hdrData.insert(bucket.hdrData.end(), header.begin(), header.end()); + bucket.hdrData.push_back('\0'); + + bucket.written++; + } + + for (size_t i = 0; i < buckets.size(); i++) { + BucketOutput &bucket = buckets[i]; + if (bucket.written != plan.entries[i].count) { + Debug(Debug::ERROR) << "Chunk " << plan.chunkIdx << " wrote " << bucket.written + << " sequences of length " << plan.entries[i].length + << " but pass 1 counted " << plan.entries[i].count << "\n"; + EXIT(EXIT_FAILURE); + } + if (bucket.written == 0) { + continue; + } + writeAt(seqFd, bucket.seqData.data(), bucket.seqData.size(), bucket.dataOffset, "sequence data"); + writeAt(hdrFd, bucket.hdrData.data(), bucket.hdrData.size(), bucket.hdrOffset, "header data"); + writeAt(seqIdxFd, bucket.seqIndex.data(), bucket.seqIndex.size() * sizeof(DenseIndex::Entry), + DenseIndex::entryOffset(bucket.keyStart), "sequence index"); + writeAt(hdrIdxFd, bucket.hdrIndex.data(), bucket.hdrIndex.size() * sizeof(DenseIndex::Entry), + DenseIndex::entryOffset(bucket.keyStart), "header index"); + } +} + +// Creates a file of exactly `size` bytes for the workers to pwrite into. +void allocateFile(const std::string &path, size_t size) { + FILE *file = FileUtil::openAndDelete(path.c_str(), "wb"); + if (ftruncate(fileno(file), static_cast(size)) != 0) { + Debug(Debug::ERROR) << "Cannot size " << path << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + if (fclose(file) != 0) { + Debug(Debug::ERROR) << "Cannot close " << path << "\n"; + EXIT(EXIT_FAILURE); + } +} + +// Runs a queue to completion with every thread of this worker claiming its own +// items. Leases mean a worker that dies mid-item only delays that item; another +// worker picks it up once the lease expires. +// The claim is recorded against the *process*, not the thread, because that is +// the unit that dies: if this worker is killed, every item its threads held must +// become re-claimable together once the lease expires. Passing a thread index +// here instead would also make identities collide across worker processes, since +// every process numbers its threads from zero. +template +void runQueue(WorkQueue &queue, int threads, int64_t workerId, Body body) { +#pragma omp parallel num_threads(threads) + { + while (true) { + const int64_t item = queue.claim(workerId); + if (item < 0) { + break; + } + body(static_cast(item)); + queue.complete(item, workerId); + } + } +} + +} // namespace + +int createdbparallel(int argc, const char **argv, const Command &command) { + Parameters &par = Parameters::getInstance(); + par.parseParameters(argc, argv, command, false, Parameters::PARSE_VARIADIC, 0); + par.printParameters(command.cmd, argc, argv, *command.params); + + std::vector filenames(par.filenames); + const std::string dataFile = filenames.back(); + filenames.pop_back(); + + // Same ordering rule as createdb, so file numbering matches between them. + SORT_SERIAL(filenames.begin(), filenames.end(), [](const std::string &a, const std::string &b) { + return FileUtil::baseName(a) < FileUtil::baseName(b); + }); + for (size_t i = 0; i < filenames.size(); i++) { + if (FileUtil::directoryExists(filenames[i].c_str()) == true) { + Debug(Debug::ERROR) << "File " << filenames[i] << " is a directory\n"; + EXIT(EXIT_FAILURE); + } + } + + const std::string hdrDataFile = dataFile + "_h"; + // Derived, not passed, so every worker runs a byte-identical command line. + const std::string coordDir = dataFile + ".coord"; + if (FileUtil::directoryExists(coordDir.c_str()) == false) { + FileUtil::makeDir(coordDir.c_str()); + } + + const std::vector chunks = planChunks(filenames, par.chunkSize); + if (chunks.empty()) { + Debug(Debug::ERROR) << "The input files have no entry\n"; + EXIT(EXIT_FAILURE); + } + + SharedCounter workerCounter(coordDir + "/worker.counter"); + const int64_t workerId = workerCounter.fetchAdd(); + Debug(Debug::INFO) << "Worker " << workerId << " joined, " << chunks.size() << " chunks\n"; + + // Pass 1: histogram every chunk. + { + WorkQueue scanQueue(coordDir + "/scan.queue", static_cast(chunks.size())); + runQueue(scanQueue, par.threads, workerId, [&](size_t chunkIdx) { + const std::string path = chunkHistPath(coordDir, chunkIdx); + if (FileUtil::fileExists(path.c_str()) == true) { + return; + } + ChunkHistogram histogram = scanChunk(filenames[chunks[chunkIdx].fileIdx], + chunks[chunkIdx], chunkIdx); + histogram.write(path); + }); + if (scanQueue.awaitAll() == false) { + Debug(Debug::ERROR) << "Scan pass stalled: work remains but no item is claimable\n"; + EXIT(EXIT_FAILURE); + } + } + Debug(Debug::INFO) << "Scan pass done\n"; + + // Plan: one worker turns the histograms into placements and lays out the + // output files. The rest wait on the sentinel. + const std::string planDone = coordDir + "/plan.done"; + { + FileLock planLock(coordDir + "/plan.lock"); + planLock.lock(); + if (FileUtil::fileExists(planDone.c_str()) == false) { + std::vector histograms; + histograms.reserve(chunks.size()); + for (size_t i = 0; i < chunks.size(); i++) { + histograms.push_back(ChunkHistogram::read(chunkHistPath(coordDir, i))); + } + + std::vector plans; + const LengthRankedTotals totals = buildLengthRankedPlan(histograms, plans); + if (totals.seqCount == 0) { + Debug(Debug::ERROR) << "The input files have no entry\n"; + EXIT(EXIT_FAILURE); + } + for (size_t i = 0; i < plans.size(); i++) { + plans[i].write(chunkPlanPath(coordDir, plans[i].chunkIdx)); + } + + allocateFile(dataFile, totals.dataBytes); + allocateFile(hdrDataFile, totals.headerBytes); + DenseIndex::createEmpty(dataFile, totals.seqCount, 0, totals.dataBytes, + static_cast(totals.maxSeqLen + 2)); + DenseIndex::createEmpty(hdrDataFile, totals.seqCount, 0, totals.headerBytes, 0); + + const std::string sourceFile = dataFile + ".source"; + FILE *source = FileUtil::openAndDelete(sourceFile.c_str(), "w"); + for (size_t i = 0; i < filenames.size(); i++) { + fprintf(source, "%zu\t%s\n", i, FileUtil::baseName(filenames[i]).c_str()); + } + if (fclose(source) != 0) { + Debug(Debug::ERROR) << "Cannot close " << sourceFile << "\n"; + EXIT(EXIT_FAILURE); + } + + // The database type is decided from votes across the whole input, not + // from whichever chunk happened to be scanned first. + int dbType = Parameters::DBTYPE_AMINO_ACIDS; + if (par.dbType == 2) { + dbType = Parameters::DBTYPE_NUCLEOTIDES; + } else if (par.dbType == 0 && totals.sampleCount > 0 && + totals.nuclVotes == totals.sampleCount) { + dbType = Parameters::DBTYPE_NUCLEOTIDES; + } + FileUtil::writeFile(coordDir + "/dbtype", + reinterpret_cast(&dbType), sizeof(int)); + + Debug(Debug::INFO) << "Planned " << totals.seqCount << " sequences, " + << totals.dataBytes << " data bytes, longest " << totals.maxSeqLen << "\n"; + FILE *sentinel = FileUtil::openAndDelete(planDone.c_str(), "w"); + fclose(sentinel); + } + planLock.unlock(); + } + + // Pass 2: write the sequences. + { + const int seqFd = openForWrite(dataFile); + const int hdrFd = openForWrite(hdrDataFile); + const int seqIdxFd = openForWrite(DenseIndex::fileName(dataFile)); + const int hdrIdxFd = openForWrite(DenseIndex::fileName(hdrDataFile)); + + WorkQueue emitQueue(coordDir + "/emit.queue", static_cast(chunks.size())); + runQueue(emitQueue, par.threads, workerId, [&](size_t chunkIdx) { + const ChunkPlan plan = ChunkPlan::read(chunkPlanPath(coordDir, chunkIdx)); + emitChunk(filenames[chunks[chunkIdx].fileIdx], chunks[chunkIdx], plan, + seqFd, hdrFd, seqIdxFd, hdrIdxFd); + }); + if (emitQueue.awaitAll() == false) { + Debug(Debug::ERROR) << "Emit pass stalled: work remains but no item is claimable\n"; + EXIT(EXIT_FAILURE); + } + + // fsync before the sentinel, so a worker that finalises after a crash + // cannot read a partially flushed database. + fsync(seqFd); + fsync(hdrFd); + fsync(seqIdxFd); + fsync(hdrIdxFd); + close(seqFd); + close(hdrFd); + close(seqIdxFd); + close(hdrIdxFd); + } + Debug(Debug::INFO) << "Emit pass done\n"; + + // Finalise: one worker writes the type files and the text indices that the + // stock tools still expect. The distributed stages read the dense index. + const std::string finalizeDone = coordDir + "/finalize.done"; + { + FileLock finalizeLock(coordDir + "/finalize.lock"); + finalizeLock.lock(); + if (FileUtil::fileExists(finalizeDone.c_str()) == false) { + int dbType = Parameters::DBTYPE_AMINO_ACIDS; + FILE *typeFile = FileUtil::openFileOrDie((coordDir + "/dbtype").c_str(), "rb", true); + if (fread(&dbType, sizeof(int), 1, typeFile) != 1) { + Debug(Debug::ERROR) << "Cannot read the planned database type\n"; + EXIT(EXIT_FAILURE); + } + fclose(typeFile); + + DBWriter::writeDbtypeFile(dataFile.c_str(), dbType, par.compressed); + DBWriter::writeDbtypeFile(hdrDataFile.c_str(), Parameters::DBTYPE_GENERIC_DB, par.compressed); + DenseIndex::writeTextIndex(dataFile); + DenseIndex::writeTextIndex(hdrDataFile); + + Debug(Debug::INFO) << "Database type: " << Parameters::getDbTypeName(dbType) << "\n"; + FILE *sentinel = FileUtil::openAndDelete(finalizeDone.c_str(), "w"); + fclose(sentinel); + } + finalizeLock.unlock(); + } + + return EXIT_SUCCESS; +} From 53efff49b6e61258a9a57c43090a51ce8283c545 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Wed, 29 Jul 2026 11:14:01 +0000 Subject: [PATCH 02/27] Add distributed k-mer shuffle. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Björn Buschkämper --- src/CommandDeclarations.h | 1 + src/MMseqsBase.cpp | 12 + src/commons/ParallelCoordination.h | 47 ++++ src/commons/Parameters.cpp | 28 +++ src/commons/Parameters.h | 3 + src/linclust/CMakeLists.txt | 1 + src/linclust/KmerPartition.cpp | 100 +++++++- src/linclust/KmerPartition.h | 116 +++++++-- src/linclust/kmermatcher.cpp | 106 ++++++--- src/linclust/kmermatcher.h | 10 +- src/linclust/kmermatcherparallel.cpp | 338 +++++++++++++++++++++++++++ src/test/TestKmerPartition.cpp | 77 +++++- src/util/createdbparallel.cpp | 28 +-- 13 files changed, 803 insertions(+), 64 deletions(-) create mode 100644 src/linclust/kmermatcherparallel.cpp diff --git a/src/CommandDeclarations.h b/src/CommandDeclarations.h index acc8b8042..195d2a122 100644 --- a/src/CommandDeclarations.h +++ b/src/CommandDeclarations.h @@ -27,6 +27,7 @@ extern int convertmsa(int argc, const char **argv, const Command& command); extern int convertprofiledb(int argc, const char **argv, const Command& command); extern int createdb(int argc, const char **argv, const Command& command); extern int createdbparallel(int argc, const char **argv, const Command& command); +extern int kmermatcherparallel(int argc, const char **argv, const Command& command); extern int makepaddedseqdb(int argc, const char **argv, const Command& command); extern int createindex(int argc, const char **argv, const Command& command); extern int createlinindex(int argc, const char **argv, const Command& command); diff --git a/src/MMseqsBase.cpp b/src/MMseqsBase.cpp index 1f8b8cf13..e336d1eb8 100644 --- a/src/MMseqsBase.cpp +++ b/src/MMseqsBase.cpp @@ -679,6 +679,18 @@ std::vector baseCommands = { " ", CITATION_MMSEQS2,{{"sequenceDB", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::sequenceDb }, {"prefilterDB", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::prefilterDb }}}, + {"kmermatcherparallel", kmermatcherparallel, &par.kmermatcherparallel, COMMAND_PREFILTER, + "Shuffle a sequence DB's k-mers into partition buckets with many workers", + "# Scan the sequence DB once and write every k-mer into the bucket of its\n" + "# partition. Every worker runs this identical command line and coordinates\n" + "# through /coord; workers may join late, die and be restarted.\n" + "mmseqs kmermatcherparallel sequenceDB kmerDir\n\n" + "# Run it from several nodes against the same shared filesystem\n" + "srun -N 8 mmseqs kmermatcherparallel sequenceDB kmerDir --scratch-budget 100T\n", + "Martin Steinegger ", + " ", + CITATION_MMSEQS2, {{"sequenceDB", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::sequenceDb }, + {"kmerDir", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::flatfile }}}, {"kmersearch", kmersearch, &par.kmersearch, COMMAND_PREFILTER, "Find bottom-m-hashed k-mer matches between target and query DB", NULL, diff --git a/src/commons/ParallelCoordination.h b/src/commons/ParallelCoordination.h index 4a7308df7..d747a94eb 100644 --- a/src/commons/ParallelCoordination.h +++ b/src/commons/ParallelCoordination.h @@ -2,6 +2,8 @@ #define MMSEQS_PARALLELCOORDINATION_H #include +#include +#include #include #include @@ -128,8 +130,53 @@ class WorkQueue { // no item was claimable for stallSeconds, which means the remaining work is // held by workers that are gone but whose leases have not yet expired, or // that the run is genuinely stuck. + // + // Prefer drain(): waiting here does not re-claim, so a worker that reaches + // this call while a crashed worker's lease is still live will wait for items + // that nobody is going to finish. bool awaitAll(unsigned int pollSeconds = 5, unsigned int stallSeconds = 0); + // Claims items and runs body on each until every item in the queue is DONE. + // + // The loop re-enters claiming after waiting, which is the whole point: when + // claim() returns -1 the work is not finished, it merely means every unfinished + // item is held under a live lease. If the workers holding them are alive, they + // will complete them; if they died, the leases expire and the items become this + // worker's to redo. Stopping at the first -1 instead -- claim, then wait for + // the rest -- leaves a crashed worker's items unfinished forever, since nobody + // is left to complete them and waiting alone never re-claims. + // + // Returns false if no item completed anywhere in the run for stallSeconds, + // which by then means the run really is stuck rather than merely waiting on a + // lease. + template + bool drain(int64_t workerId, Body body, unsigned int pollSeconds = 5, + unsigned int stallSeconds = 2 * DEFAULT_LEASE_SECONDS) { + int64_t lastDone = -1; + int64_t lastProgress = static_cast(time(NULL)); + while (true) { + const int64_t item = claim(workerId); + if (item >= 0) { + body(static_cast(item)); + complete(item, workerId); + continue; + } + const int64_t done = getDoneCount(); + if (done >= itemCount) { + return true; + } + if (done != lastDone) { + lastDone = done; + lastProgress = static_cast(time(NULL)); + } else if (stallSeconds > 0 && + static_cast(time(NULL)) - lastProgress > + static_cast(stallSeconds)) { + return false; + } + sleep(pollSeconds); + } + } + static const int64_t DEFAULT_LEASE_SECONDS = 1800; private: diff --git a/src/commons/Parameters.cpp b/src/commons/Parameters.cpp index f113406cc..64f12c8e0 100644 --- a/src/commons/Parameters.cpp +++ b/src/commons/Parameters.cpp @@ -56,6 +56,7 @@ Parameters::Parameters(): PARAM_SPLIT_MODE(PARAM_SPLIT_MODE_ID, "--split-mode", "Split mode", "0: split target db; 1: split query db; 2: auto, depending on main memory", typeid(int), (void *) &splitMode, "^[0-2]{1}$", MMseqsParameter::COMMAND_PREFILTER | MMseqsParameter::COMMAND_EXPERT), PARAM_SPLIT_MEMORY_LIMIT(PARAM_SPLIT_MEMORY_LIMIT_ID, "--split-memory-limit", "Split memory limit", "Set max memory per split. E.g. 800B, 5K, 10M, 1G. Default (0) to all available system memory", typeid(ByteParser), (void *) &splitMemoryLimit, "^(0|[1-9]{1}[0-9]*(B|K|M|G|T)?)$", MMseqsParameter::COMMAND_COMMON | MMseqsParameter::COMMAND_PREFILTER | MMseqsParameter::COMMAND_EXPERT), PARAM_CHUNK_SIZE(PARAM_CHUNK_SIZE_ID, "--chunk-size", "Chunk size", "Input bytes one work item covers. Smaller chunks spread work more evenly and use less memory per thread; larger chunks mean fewer coordination files. E.g. 64M, 256M, 1G", typeid(ByteParser), (void *) &chunkSize, "^([1-9]{1}[0-9]*(B|K|M|G|T)?)$", MMseqsParameter::COMMAND_EXPERT), + PARAM_SCRATCH_BUDGET(PARAM_SCRATCH_BUDGET_ID, "--scratch-budget", "Scratch budget", "Total scratch the run may occupy. The k-mer extraction wave count and the partition count are derived from this together with --split-memory-limit, rather than set by hand. Default (0) for a single wave. E.g. 100T, 500T", typeid(ByteParser), (void *) &scratchBudget, "^(0|[1-9]{1}[0-9]*(B|K|M|G|T)?)$", MMseqsParameter::COMMAND_EXPERT), PARAM_DISK_SPACE_LIMIT(PARAM_DISK_SPACE_LIMIT_ID, "--disk-space-limit", "Disk space limit", "Set max disk space to use for reverse profile searches. E.g. 800B, 5K, 10M, 1G. Default (0) to all available disk space in the temp folder", typeid(ByteParser), (void *) &diskSpaceLimit, "^(0|[1-9]{1}[0-9]*(B|K|M|G|T)?)$", MMseqsParameter::COMMAND_COMMON | MMseqsParameter::COMMAND_PREFILTER | MMseqsParameter::COMMAND_EXPERT), PARAM_SPLIT_AMINOACID(PARAM_SPLIT_AMINOACID_ID, "--split-aa", "Split by amino acid", "Try to find the best split boundaries by entry lengths", typeid(bool), (void *) &splitAA, "$", MMseqsParameter::COMMAND_EXPERT), PARAM_SUB_MAT(PARAM_SUB_MAT_ID, "--sub-mat", "Substitution matrix", "Substitution matrix file", typeid(MultiParam>), (void *) &scoringMatrixFile, "", MMseqsParameter::COMMAND_COMMON | MMseqsParameter::COMMAND_EXPERT), @@ -920,6 +921,32 @@ Parameters::Parameters(): createdbparallel.push_back(&PARAM_COMPRESSED); createdbparallel.push_back(&PARAM_V); + // kmermatcherparallel + // Extraction knobs only: this stage stops at the shuffled k-mer buckets, so + // the grouping and alignment parameters belong to the reduce that reads them. + // No PARAM_ADJUST_KMER_LEN either -- the adjusted length is a property of the + // whole database that a worker scanning one key range cannot agree on. + kmermatcherparallel.push_back(&PARAM_SUB_MAT); + kmermatcherparallel.push_back(&PARAM_ALPH_SIZE); + kmermatcherparallel.push_back(&PARAM_MIN_SEQ_ID); + kmermatcherparallel.push_back(&PARAM_K); + kmermatcherparallel.push_back(&PARAM_KMER_PER_SEQ); + kmermatcherparallel.push_back(&PARAM_KMER_PER_SEQ_SCALE); + kmermatcherparallel.push_back(&PARAM_SPACED_KMER_MODE); + kmermatcherparallel.push_back(&PARAM_SPACED_KMER_PATTERN); + kmermatcherparallel.push_back(&PARAM_MASK_RESIDUES); + kmermatcherparallel.push_back(&PARAM_MASK_PROBABILTY); + kmermatcherparallel.push_back(&PARAM_MASK_LOWER_CASE); + kmermatcherparallel.push_back(&PARAM_MASK_N_REPEAT); + kmermatcherparallel.push_back(&PARAM_MAX_SEQ_LEN); + kmermatcherparallel.push_back(&PARAM_HASH_SHIFT); + kmermatcherparallel.push_back(&PARAM_IGNORE_MULTI_KMER); + kmermatcherparallel.push_back(&PARAM_SPLIT_MEMORY_LIMIT); + kmermatcherparallel.push_back(&PARAM_SCRATCH_BUDGET); + kmermatcherparallel.push_back(&PARAM_THREADS); + kmermatcherparallel.push_back(&PARAM_COMPRESSED); + kmermatcherparallel.push_back(&PARAM_V); + // makepaddedseqdb makepaddedseqdb.push_back(&PARAM_SUB_MAT); makepaddedseqdb.push_back(&PARAM_SCORE_BIAS); @@ -2511,6 +2538,7 @@ void Parameters::setDefaults() { splitMode = DETECT_BEST_DB_SPLIT; splitMemoryLimit = 0; chunkSize = 256 * 1024 * 1024; + scratchBudget = 0; diskSpaceLimit = 0; splitAA = false; spacedKmerPattern = ""; diff --git a/src/commons/Parameters.h b/src/commons/Parameters.h index 18d0b5b0f..63d255f93 100644 --- a/src/commons/Parameters.h +++ b/src/commons/Parameters.h @@ -440,6 +440,7 @@ class Parameters { int splitMode; // Split by query or target DB size_t splitMemoryLimit; // Maximum memory in bytes a split can use size_t chunkSize; // Input bytes one createdbparallel work item covers + size_t scratchBudget; // Scratch ceiling the k-mer wave count is derived from size_t diskSpaceLimit; // Maximum disk space in bytes for sliced reverse profile search bool splitAA; // Split database by amino acid count instead int preloadMode; // Preload mode of database @@ -829,6 +830,7 @@ class Parameters { PARAMETER(PARAM_SPLIT_MODE) PARAMETER(PARAM_SPLIT_MEMORY_LIMIT) PARAMETER(PARAM_CHUNK_SIZE) + PARAMETER(PARAM_SCRATCH_BUDGET) PARAMETER(PARAM_DISK_SPACE_LIMIT) PARAMETER(PARAM_SPLIT_AMINOACID) PARAMETER(PARAM_SUB_MAT) @@ -1235,6 +1237,7 @@ class Parameters { std::vector convertalignments; std::vector createdb; std::vector createdbparallel; + std::vector kmermatcherparallel; std::vector makepaddedseqdb; std::vector convert2fasta; std::vector result2flat; diff --git a/src/linclust/CMakeLists.txt b/src/linclust/CMakeLists.txt index 2dbdf23ae..14bca716e 100644 --- a/src/linclust/CMakeLists.txt +++ b/src/linclust/CMakeLists.txt @@ -1,5 +1,6 @@ set(linclust_source_files linclust/kmermatcher.cpp + linclust/kmermatcherparallel.cpp linclust/KmerPartition.cpp linclust/kmerindexdb.cpp linclust/kmersearch.cpp diff --git a/src/linclust/KmerPartition.cpp b/src/linclust/KmerPartition.cpp index 2516d47a2..92a90c5d1 100644 --- a/src/linclust/KmerPartition.cpp +++ b/src/linclust/KmerPartition.cpp @@ -23,11 +23,67 @@ KmerPartitioner::KmerPartitioner(unsigned int partitionCount) : partitionCount(p << " exceeds the 16-bit k-mer hash space (max 65536)\n"; EXIT(EXIT_FAILURE); } - unsigned int bits = 0; - while ((1u << bits) < partitionCount) { - bits++; + mask = partitionCount - 1; +} + +namespace { + +uint64_t divideRoundingUp(uint64_t value, uint64_t divisor) { + return (value + divisor - 1) / divisor; +} + +unsigned int roundUpToPowerOfTwo(uint64_t value) { + unsigned int result = 1; + while (result < value) { + result <<= 1; } - shift = 16 - bits; + return result; +} + +} // namespace + +KmerShuffleSizing deriveKmerShuffleSizing(uint64_t sequenceCount, unsigned int kmersPerSequence, + uint64_t scratchBudgetBytes, uint64_t persistentBytes, + uint64_t workerMemoryBytes) { + KmerShuffleSizing sizing; + sizing.totalKmerBytes = sequenceCount * kmersPerSequence * sizeof(KmerRecord); + + if (scratchBudgetBytes == 0) { + // No budget given: one wave, and let the memory constraint alone pick P. + sizing.waveCount = 1; + } else { + if (persistentBytes >= scratchBudgetBytes) { + Debug(Debug::ERROR) << "Scratch budget of " << scratchBudgetBytes + << " bytes is already exhausted by the sequence database and " + << "surviving edges (" << persistentBytes + << " bytes). No wave count can fit the k-mer shuffle.\n"; + EXIT(EXIT_FAILURE); + } + const uint64_t available = scratchBudgetBytes - persistentBytes; + sizing.waveCount = static_cast( + std::max(divideRoundingUp(sizing.totalKmerBytes, available), 1)); + } + sizing.bytesPerWave = divideRoundingUp(sizing.totalKmerBytes, sizing.waveCount); + + if (workerMemoryBytes == 0) { + sizing.partitionCount = 1; + } else { + sizing.partitionCount = + roundUpToPowerOfTwo(divideRoundingUp(sizing.bytesPerWave, workerMemoryBytes)); + } + if (sizing.partitionCount > 65536) { + // Above the 16-bit hash space the partitioner cannot tell partitions + // apart, so this is a real dead end rather than something to clamp: the + // run needs a bigger node, more waves, or a smaller input. + Debug(Debug::ERROR) << "A wave holds " << sizing.bytesPerWave << " bytes of k-mers, which " + << "needs more than 65536 partitions to fit " << workerMemoryBytes + << " bytes per worker. The 16-bit k-mer hash cannot address that many. " + << "Increase the per-worker memory or lower the scratch budget so more " + << "waves are used.\n"; + EXIT(EXIT_FAILURE); + } + sizing.bytesPerPartition = divideRoundingUp(sizing.bytesPerWave, sizing.partitionCount); + return sizing; } std::string KmerBucketWriter::partitionDir(const std::string &dir, unsigned int partition) { @@ -54,7 +110,8 @@ void KmerBucketWriter::createLayout(const std::string &dir, unsigned int partiti KmerBucketWriter::KmerBucketWriter(const std::string &dir, unsigned int partitionCount, const std::string &shardId, size_t bufferBudgetBytes) - : dir(dir), shardId(shardId), partitionCount(partitionCount), recordCount(0) { + : dir(dir), shardId(shardId), partitionCount(partitionCount), + mutexes(partitionCount) { // At least a handful of records per partition even with a tiny budget, so a // large partition count degrades to more frequent flushes rather than to // one write syscall per k-mer. @@ -62,6 +119,7 @@ KmerBucketWriter::KmerBucketWriter(const std::string &dir, unsigned int partitio recordsPerBuffer = std::max(perPartition, 16); buffers.resize(partitionCount); files.assign(partitionCount, NULL); + recordCounts.assign(partitionCount, 0); } KmerBucketWriter::~KmerBucketWriter() { @@ -84,22 +142,50 @@ void KmerBucketWriter::flush(unsigned int partition) { } } if (fwrite(buffer.data(), sizeof(KmerRecord), buffer.size(), files[partition]) != buffer.size()) { - Debug(Debug::ERROR) << "Cannot write k-mer bucket for partition " << partition << "\n"; + // Name the reason: a full scratch filesystem is by far the likeliest way + // this stage fails, and "cannot write" alone sends you looking for a bug. + Debug(Debug::ERROR) << "Cannot write " << buffer.size() << " k-mer records to bucket " + << partition << " of " << dir << ": " << strerror(errno) << "\n"; EXIT(EXIT_FAILURE); } buffer.clear(); } void KmerBucketWriter::append(unsigned int partition, const KmerRecord &record) { + std::lock_guard guard(mutexes[partition]); buffers[partition].push_back(record); if (buffers[partition].size() >= recordsPerBuffer) { flush(partition); } - recordCount++; + // Counted per partition rather than in one shared counter: this runs on every + // k-mer, and a single counter would be the one point every thread contends on. + recordCounts[partition]++; +} + +uint64_t KmerBucketWriter::getRecordCount() { + uint64_t total = 0; + for (unsigned int p = 0; p < partitionCount; p++) { + std::lock_guard guard(mutexes[p]); + total += recordCounts[p]; + } + return total; +} + +void KmerBucketWriter::flushAll() { + for (unsigned int p = 0; p < partitionCount; p++) { + std::lock_guard guard(mutexes[p]); + flush(p); + if (files[p] != NULL && fflush(files[p]) != 0) { + Debug(Debug::ERROR) << "Cannot flush k-mer bucket for partition " << p << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + } } void KmerBucketWriter::close() { for (unsigned int p = 0; p < partitionCount; p++) { + std::lock_guard guard(mutexes[p]); flush(p); if (files[p] != NULL) { if (fclose(files[p]) != 0) { diff --git a/src/linclust/KmerPartition.h b/src/linclust/KmerPartition.h index 7aaa9eb48..690d51324 100644 --- a/src/linclust/KmerPartition.h +++ b/src/linclust/KmerPartition.h @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -37,16 +38,27 @@ class KmerPartitioner { unsigned int getPartitionCount() const { return partitionCount; } // Partition of a k-mer, from the 16-bit hash kmermatcher already computed. - // Taking the high bits keeps whole contiguous runs of the score space - // together, which is the same space the stock split ranges carve up, so a - // partition is directly comparable to a stock split. + // + // The *low* bits, not the high ones. Linclust keeps the bottom-scoring k-mers + // of each sequence (kmermatcher.cpp:318, `score < threshold`), so the selected + // scores are not spread over the 16-bit space at all -- they pile up against + // zero. Partitioning on the high bits therefore puts almost everything in + // partition 0: measured on 1M sequences at P=16 it gave partition 0 68% of all + // k-mers and partition 15 0.3%, which would make the reduce's per-partition + // memory bound meaningless. The selected scores form a prefix [0, threshold) + // of the space, and the low bits of a prefix are uniform, so masking balances + // the partitions without needing the hash-distribution pass stock's + // setupKmerSplits runs to carve its uneven split ranges. + // + // Any pure function of the score would be *correct*; only balance picks + // between them, since equal k-mers always share a score. unsigned int partitionOf(unsigned short score) const { - return static_cast(score) >> shift; + return static_cast(score) & mask; } private: unsigned int partitionCount; - unsigned int shift; + unsigned int mask; }; // One k-mer occurrence as stored in a bucket file. @@ -85,6 +97,42 @@ struct __attribute__((__packed__)) KmerRecord { static const uint64_t MAX_ID = (static_cast(1) << 48) - 1; }; +// The two numbers that shape the k-mer shuffle, and where they came from. +struct KmerShuffleSizing { + uint64_t totalKmerBytes; // every k-mer record, summed over all waves + unsigned int waveCount; // extraction passes, so peak scratch stays in budget + unsigned int partitionCount; // P + uint64_t bytesPerWave; // peak k-mer bytes on disk at any one time + uint64_t bytesPerPartition; // what one worker loads in the reduce +}; + +// Derives the wave count and P from the scratch budget and per-worker memory, +// rather than making either a knob. +// +// Both are over-determined by things already known: the k-mer volume follows +// from the sequence count at a measured 21 k-mers x 24 B per sequence, the wave +// count is whatever keeps peak scratch inside the budget, and P is the smallest +// power of two whose buckets still fit a worker. Exposing them raw invites two +// failures that only show up deep into a run -- P too small and workers die of +// memory, P too large and the reduce silently pays P/W sequential re-scans of +// the whole database for no benefit. +// +// The two scales this is sized for pull in opposite directions and land in +// different places, which is exactly why it should be computed: +// 100B 50.4 TB of k-mers, P = 1024, ~49 GB buckets, ~2 DB scans at W=500 +// 1T 504 TB of k-mers, P = 8192, ~62 GB buckets, ~17 DB scans at W=500 +// +// persistentBytes is everything sharing the scratch budget with the k-mer wave: +// the sequence database, which persists, plus the surviving edges the fused +// group+align stage accumulates. +// +// scratchBudgetBytes == 0 means unlimited, giving a single wave. +KmerShuffleSizing deriveKmerShuffleSizing(uint64_t sequenceCount, + unsigned int kmersPerSequence, + uint64_t scratchBudgetBytes, + uint64_t persistentBytes, + uint64_t workerMemoryBytes); + // Converts a bucket record into the KmerPosition that assignGroup consumes, and // back. Templated rather than including kmermatcher.h so this header stays free // of the whole DBReader/Parameters chain; the caller instantiates it with the @@ -119,26 +167,54 @@ void kmerPositionToRecord(KmerPositionT &in, KmerRecord &out) { // Appends k-mer records into per-partition bucket files. // -// Each writer owns one shard id and writes /p/.kmers, so -// concurrent writers -- threads within a worker, and workers across nodes -- -// never touch the same file and need no locking at all. Bucket files are laid -// out one directory per partition because the reduce stage reads exactly one -// partition, and because a flat directory would hold partitions x shards entries. +// One writer per worker *process*, shared by all its threads, writing +// /p/.kmers. Two constraints pick that granularity: +// +// - File count. A writer per thread would leave workers x threads x partitions +// shards -- 32 million at 500 workers, 64 threads and P = 1024 -- which no +// shared filesystem enjoys. Per process it is workers x partitions, so 0.5 M +// at the 100B target and 4 M at 1T, a few hundred files per directory. +// - Write size. The buffer budget is split across partitions, so per process a +// partition gets budget/P; per thread it would get budget/(threads x P), +// which at these partition counts is a few hundred bytes -- the write size +// that makes a parallel filesystem collapse. +// +// Threads therefore share the per-partition buffers under a per-partition mutex. +// There are P independent locks and the k-mer hash spreads threads uniformly +// across them, so the expected contention is threads/P, and the k-mer extraction +// that produces each record costs far more than the uncontended acquire. +// +// Workers still never share a file, so nothing is locked across nodes. Bucket +// files are laid out one directory per partition because the reduce stage reads +// exactly one partition, and because a flat directory would hold partitions x +// shards entries. class KmerBucketWriter { public: // bufferBudgetBytes is split evenly across partitions, so memory is bounded // regardless of partition count. Records accumulate per partition and are // flushed in large contiguous appends rather than one write per k-mer. KmerBucketWriter(const std::string &dir, unsigned int partitionCount, - const std::string &shardId, size_t bufferBudgetBytes = 32 * 1024 * 1024); + const std::string &shardId, size_t bufferBudgetBytes = 1024 * 1024 * 1024); ~KmerBucketWriter(); + // Thread-safe. void append(unsigned int partition, const KmerRecord &record); + + // Pushes every buffered record out to the operating system. + // + // Must be called before a work item is marked done. Buffers span items, so + // without it an item can be recorded as complete while its k-mers are still + // in memory; a worker that then dies loses them, and because the item is + // already done nobody redoes it. This is the same durability the work queue + // itself has -- both survive the process dying, neither survives the node + // going down, which is the level at which the whole stage restarts anyway. + void flushAll(); // Flushes and closes every open bucket. Called by the destructor, but call it // explicitly to see write errors before the object goes away. void close(); - uint64_t getRecordCount() const { return recordCount; } + // Records appended by this writer. Call it when the threads are quiescent. + uint64_t getRecordCount(); // Creates the per-partition directories. Safe to call from many workers. static void createLayout(const std::string &dir, unsigned int partitionCount); @@ -148,6 +224,7 @@ class KmerBucketWriter { KmerBucketWriter(const KmerBucketWriter &); KmerBucketWriter &operator=(const KmerBucketWriter &); + // Caller must hold mutexes[partition]. void flush(unsigned int partition); std::string dir; @@ -156,7 +233,8 @@ class KmerBucketWriter { size_t recordsPerBuffer; std::vector > buffers; std::vector files; - uint64_t recordCount; + std::vector mutexes; + std::vector recordCounts; }; // Reads every shard of one partition back. @@ -167,6 +245,18 @@ class KmerBucketReader { static uint64_t countRecords(const std::string &dir, unsigned int partition); // Appends every record of the partition to out. + // + // The reduce must drop exact duplicate records after sorting. A worker that + // dies mid-item has already flushed part of that item into its own shard, and + // the worker that redoes the item once the lease expires writes the same + // records into a *different* shard, so the partition can legitimately contain + // a record twice. Per-item shard files would make the map idempotent instead, + // but that is items x partitions files -- 8e9 at 1e12 sequences -- so the + // duplicates are removed here rather than prevented there. + // + // Dropping them is exact, not a heuristic: (kmer, id, pos) identifies one + // k-mer occurrence in one sequence, so two byte-identical records can only + // come from the same occurrence being written twice. static void readPartition(const std::string &dir, unsigned int partition, std::vector &out); diff --git a/src/linclust/kmermatcher.cpp b/src/linclust/kmermatcher.cpp index ccae752c6..6c042a2fa 100644 --- a/src/linclust/kmermatcher.cpp +++ b/src/linclust/kmermatcher.cpp @@ -79,10 +79,34 @@ static void flushKmerBuffer(KmerPosition *km } } +// Copies a fully staged k-mer slot into a bucket record and appends it to the +// partition its hash selects. +// +// Reading the length from the sequence rather than from the slot is deliberate: +// seqLen only round-trips through KmerPosition when IncludeSeqLen is true, but +// the record must carry it for every instantiation, since carrying it is exactly +// what deletes the key-space-sized seqkey_to_len array. +template +static void appendStagedKmerToBucket(KmerBucketWriter *writer, const KmerPartitioner &partitioner, + unsigned short score, KmerPositionT &staged, + DBKeyType seqId, int seqLen) { + KmerRecord record; + record.kmer = staged.kmer; + record.setId(static_cast(seqId)); + record.pos = static_cast(staged.pos); + record.seqLen = static_cast(seqLen); + for (int i = 0; i < 6; i++) { + record.adjacent[i] = staged.getAdjacentSeq(i); + } + writer->append(partitioner.partitionOf(score), record); +} + template std::pair fillKmerPositionArray(KmerPosition * kmerArray, size_t kmerArraySize, DBReader &seqDbr, Parameters & par, BaseMatrix * subMat, bool hashWholeSequence, - size_t hashStartRange, size_t hashEndRange, size_t * hashDistribution){ + size_t hashStartRange, size_t hashEndRange, size_t * hashDistribution, + KmerBucketWriter *bucketWriter, + const KmerPartitioner *kmerPartitioner){ size_t offset = 0; int querySeqType = seqDbr.getDbtype(); size_t longestKmer = par.kmerSize; @@ -267,10 +291,19 @@ std::pair fillKmerPositionArray(KmerPosition= BUFFER_SIZE) { - flushKmerBuffer(kmerArray, kmerArraySize, threadKmerBuffer, bufferPos, &offset); - bufferPos = 0; + if (bucketWriter != NULL) { + // The identity k-mer is partitioned by the same value the split + // range test above uses, the low 16 bits of the sequence hash, so + // two identical sequences always meet in the same partition. + appendStagedKmerToBucket(bucketWriter, *kmerPartitioner, + static_cast(seqHash), + threadKmerBuffer[bufferPos], seqId, seq.L); + } else { + bufferPos++; + if (bufferPos >= BUFFER_SIZE) { + flushKmerBuffer(kmerArray, kmerArraySize, threadKmerBuffer, bufferPos, &offset); + bufferPos = 0; + } } } } @@ -319,7 +352,10 @@ std::pair fillKmerPositionArray(KmerPositionscore >= hashStartRange && (kmers + kmerIdx)->score <= hashEndRange) + const bool keepKmer = (bucketWriter != NULL) + || ((kmers + kmerIdx)->score >= hashStartRange + && (kmers + kmerIdx)->score <= hashEndRange); + if (keepKmer) { if(hashDistribution != NULL){ __sync_fetch_and_add(&hashDistribution[(kmers + kmerIdx)->score], 1); @@ -358,11 +394,18 @@ std::pair fillKmerPositionArray(KmerPosition= BUFFER_SIZE) { - flushKmerBuffer(kmerArray, kmerArraySize, threadKmerBuffer, bufferPos, &offset); - bufferPos = 0; + if (bucketWriter != NULL) { + appendStagedKmerToBucket(bucketWriter, *kmerPartitioner, + (kmers + kmerIdx)->score, + threadKmerBuffer[bufferPos], seqId, seq.L); + // The staging slot is reused; kmerArray stays untouched. + } else { + bufferPos++; + + if (bufferPos >= BUFFER_SIZE) { + flushKmerBuffer(kmerArray, kmerArraySize, threadKmerBuffer, bufferPos, &offset); + bufferPos = 0; + } } } } @@ -2066,23 +2109,34 @@ void setKmerLengthAndAlphabet(Parameters ¶meters, size_t aaDbSize, int seqTy } // Existing explicit instantiations (IncludeSeqLen defaults to false) -template std::pair fillKmerPositionArray<0, short, true>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *); -template std::pair fillKmerPositionArray<0, short, false>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *); -template std::pair fillKmerPositionArray<1, short, true>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *); -template std::pair fillKmerPositionArray<1, short, false>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *); -template std::pair fillKmerPositionArray<2, short, true>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *); -template std::pair fillKmerPositionArray<2, short, false>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *); -template std::pair fillKmerPositionArray<0, int, true>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *); -template std::pair fillKmerPositionArray<0, int, false>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *); -template std::pair fillKmerPositionArray<1, int, true>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *); -template std::pair fillKmerPositionArray<1, int, false>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *); -template std::pair fillKmerPositionArray<2, int, true>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *); -template std::pair fillKmerPositionArray<2, int, false>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *); +template std::pair fillKmerPositionArray<0, short, true>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *, KmerBucketWriter *, const KmerPartitioner *); +template std::pair fillKmerPositionArray<0, short, false>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *, KmerBucketWriter *, const KmerPartitioner *); +template std::pair fillKmerPositionArray<1, short, true>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *, KmerBucketWriter *, const KmerPartitioner *); +template std::pair fillKmerPositionArray<1, short, false>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *, KmerBucketWriter *, const KmerPartitioner *); +template std::pair fillKmerPositionArray<2, short, true>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *, KmerBucketWriter *, const KmerPartitioner *); +template std::pair fillKmerPositionArray<2, short, false>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *, KmerBucketWriter *, const KmerPartitioner *); +template std::pair fillKmerPositionArray<0, int, true>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *, KmerBucketWriter *, const KmerPartitioner *); +template std::pair fillKmerPositionArray<0, int, false>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *, KmerBucketWriter *, const KmerPartitioner *); +template std::pair fillKmerPositionArray<1, int, true>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *, KmerBucketWriter *, const KmerPartitioner *); +template std::pair fillKmerPositionArray<1, int, false>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *, KmerBucketWriter *, const KmerPartitioner *); +template std::pair fillKmerPositionArray<2, int, true>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *, KmerBucketWriter *, const KmerPartitioner *); +template std::pair fillKmerPositionArray<2, int, false>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *, KmerBucketWriter *, const KmerPartitioner *); // Linsearch explicit instantiations (IncludeSeqLen=true) -template std::pair fillKmerPositionArray<0, short, false, true>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *); -template std::pair fillKmerPositionArray<1, short, false, true>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *); -template std::pair fillKmerPositionArray<2, short, false, true>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *); +template std::pair fillKmerPositionArray<0, short, false, true>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *, KmerBucketWriter *, const KmerPartitioner *); +template std::pair fillKmerPositionArray<1, short, false, true>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *, KmerBucketWriter *, const KmerPartitioner *); +template std::pair fillKmerPositionArray<2, short, false, true>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *, KmerBucketWriter *, const KmerPartitioner *); + +// Distributed map explicit instantiations. Adjacency and inline lengths are both +// on: the bucket record always carries the flanking residues and the sequence +// length, and carrying the length is what deletes the key-space-sized +// seqkey_to_len array that cannot exist at 1e12 sequences. +template std::pair fillKmerPositionArray<0, short, true, true>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *, KmerBucketWriter *, const KmerPartitioner *); +template std::pair fillKmerPositionArray<1, short, true, true>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *, KmerBucketWriter *, const KmerPartitioner *); +template std::pair fillKmerPositionArray<2, short, true, true>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *, KmerBucketWriter *, const KmerPartitioner *); +template std::pair fillKmerPositionArray<0, int, true, true>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *, KmerBucketWriter *, const KmerPartitioner *); +template std::pair fillKmerPositionArray<1, int, true, true>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *, KmerBucketWriter *, const KmerPartitioner *); +template std::pair fillKmerPositionArray<2, int, true, true>(KmerPosition *, size_t, DBReader &, Parameters &, BaseMatrix *, bool, size_t, size_t, size_t *, KmerBucketWriter *, const KmerPartitioner *); template KmerPosition *initKmerPositionMemory(size_t size); template KmerPosition *initKmerPositionMemory(size_t size); diff --git a/src/linclust/kmermatcher.h b/src/linclust/kmermatcher.h index 1248b6e5a..589e11753 100644 --- a/src/linclust/kmermatcher.h +++ b/src/linclust/kmermatcher.h @@ -4,6 +4,7 @@ #include "DBWriter.h" #include "Parameters.h" #include "BaseMatrix.h" +#include "KmerPartition.h" #include #ifndef SIZE_T_MAX @@ -269,7 +270,14 @@ KmerPosition *initKmerPositionMemory(size_t template std::pair fillKmerPositionArray(KmerPosition * kmerArray, size_t kmerArraySize, DBReader &seqDbr, Parameters & par, BaseMatrix * subMat, bool hashWholeSequence, - size_t hashStartRange, size_t hashEndRange, size_t * hashDistribution); + size_t hashStartRange, size_t hashEndRange, size_t * hashDistribution, + // When set, every selected k-mer is appended to its partition bucket + // instead of being filtered down to [hashStartRange, hashEndRange] and + // staged into kmerArray. This is what turns the per-split full rescan + // into a single scan: nothing extracted is thrown away. + // One writer shared by every thread; it is thread-safe. + KmerBucketWriter *bucketWriter = NULL, + const KmerPartitioner *kmerPartitioner = NULL); void maskSequence(int maskMode, int maskLowerCase, diff --git a/src/linclust/kmermatcherparallel.cpp b/src/linclust/kmermatcherparallel.cpp new file mode 100644 index 000000000..de4643408 --- /dev/null +++ b/src/linclust/kmermatcherparallel.cpp @@ -0,0 +1,338 @@ +/* + * kmermatcherparallel -- the map half of the distributed linclust. + * + * Scans the sequence database once and writes every extracted k-mer into the + * bucket file of the partition its hash selects. Many worker processes run the + * same command line and coordinate only through files in a shared directory; a + * worker may join late, die, or be restarted without losing or duplicating work. + * + * Why this exists instead of running stock kmermatcher with more splits: a split + * re-reads the whole database and re-extracts every k-mer, then discards the + * ones outside its hash range (kmermatcher.cpp:322). At 1e12 sequences on a 2 TB + * node that is ~68 passes over the most expensive stage in the pipeline. Here the + * extraction is paid for exactly once and the partitioning is what carries k-mers + * to the reduce, so the cost is one scan plus one shuffle. + * + * The partitioning is lossless, not an approximation: the 16-bit score is a pure + * function of the k-mer, so equal k-mers always land in the same partition and a + * partition can be grouped in isolation (see KmerPartition.h). + * + * Memory per worker is bounded by the bucket buffers and the per-thread k-mer + * scratch, not by the database size: the database is read through DenseIndex in + * key ranges, so no worker ever holds an index entry per sequence. + */ +#include "Command.h" +#include "Debug.h" +#include "DBReader.h" +#include "DenseIndex.h" +#include "FileUtil.h" +#include "KmerPartition.h" +#include "NucleotideMatrix.h" +#include "ParallelCoordination.h" +#include "Parameters.h" +#include "ReducedMatrix.h" +#include "SubstitutionMatrix.h" +#include "Util.h" +#include "kmermatcher.h" + +#include +#include +#include +#include + +namespace { + +// Work items are contiguous key ranges, so an item is also one contiguous read +// of the data file. Their size is derived rather than fixed: a fixed size that +// suits 1e12 sequences leaves a 1e6-sequence database with a single item, which +// no second worker can ever claim. +// +// Aim for many more items than there will plausibly be workers, so late joiners +// find work and a dead worker costs one item; bound the size from below so the +// fixed per-item cost is amortised, and from above so the largest runs keep item +// counts and lease traffic sane. +// +// The floor is the load-bearing one. An item opens a DBReader over its key range, +// which re-maps the data file, and that costs ~50 ms independent of how big the +// item is. Measured on 1M sequences: 1000-sequence items took 61 s against 7.8 s +// for the same work in one item. At 50k the fixed cost is a few percent, and at +// the scales this stage is for, items sit at the ceiling anyway. +const uint64_t TARGET_ITEM_COUNT = 4096; +const uint64_t MIN_SEQUENCES_PER_ITEM = 50000; +const uint64_t MAX_SEQUENCES_PER_ITEM = 1000000; + +uint64_t deriveSequencesPerItem(uint64_t entryCount) { + const uint64_t target = entryCount / TARGET_ITEM_COUNT; + if (target < MIN_SEQUENCES_PER_ITEM) { + return MIN_SEQUENCES_PER_ITEM; + } + if (target > MAX_SEQUENCES_PER_ITEM) { + return MAX_SEQUENCES_PER_ITEM; + } + return target; +} + +// What the first worker records so every later worker can check it is joining +// the run it thinks it is. Repartitioning halfway through -- which is what a +// changed --split-memory-limit or --scratch-budget would silently do -- would +// scatter equal k-mers across different partitions and quietly lose edges. +struct ShuffleManifest { + uint64_t entryCount; + uint64_t partitionCount; + uint64_t waveCount; + uint64_t kmerSize; + + void write(const std::string &path) const { + FILE *file = FileUtil::openAndDelete(path.c_str(), "w"); + fprintf(file, "entryCount\t%zu\n", (size_t)entryCount); + fprintf(file, "partitionCount\t%zu\n", (size_t)partitionCount); + fprintf(file, "waveCount\t%zu\n", (size_t)waveCount); + fprintf(file, "kmerSize\t%zu\n", (size_t)kmerSize); + if (fclose(file) != 0) { + Debug(Debug::ERROR) << "Cannot close " << path << "\n"; + EXIT(EXIT_FAILURE); + } + } + + static ShuffleManifest read(const std::string &path) { + ShuffleManifest manifest; + FILE *file = FileUtil::openFileOrDie(path.c_str(), "r", true); + char name[64]; + size_t value; + manifest.entryCount = 0; + manifest.partitionCount = 0; + manifest.waveCount = 0; + manifest.kmerSize = 0; + while (fscanf(file, "%63s\t%zu\n", name, &value) == 2) { + const std::string key = name; + if (key == "entryCount") { + manifest.entryCount = value; + } else if (key == "partitionCount") { + manifest.partitionCount = value; + } else if (key == "waveCount") { + manifest.waveCount = value; + } else if (key == "kmerSize") { + manifest.kmerSize = value; + } + } + fclose(file); + return manifest; + } + + void requireMatches(const ShuffleManifest &other, const std::string &path) const { + if (entryCount != other.entryCount || partitionCount != other.partitionCount || + waveCount != other.waveCount || kmerSize != other.kmerSize) { + Debug(Debug::ERROR) + << "This worker derived a different k-mer shuffle than the one already in " + << "progress (" << path << "): " << partitionCount << " partitions, " + << waveCount << " waves, k=" << kmerSize << " over " << entryCount + << " sequences, against " << other.partitionCount << " partitions, " + << other.waveCount << " waves, k=" << other.kmerSize << " over " + << other.entryCount << " sequences.\n" + << "Every worker must run the same command line on the same database.\n"; + EXIT(EXIT_FAILURE); + } + } +}; + +BaseMatrix *createSubstitutionMatrix(Parameters &par, int dbType) { + if (Parameters::isEqualDbtype(dbType, Parameters::DBTYPE_NUCLEOTIDES)) { + return new NucleotideMatrix(par.scoringMatrixFile.values.nucleotide().c_str(), 1.0, 0.0); + } + if (par.alphabetSize.values.aminoacid() == 21) { + return new SubstitutionMatrix(par.scoringMatrixFile.values.aminoacid().c_str(), 2.0, 0.0); + } + SubstitutionMatrix sMat(par.scoringMatrixFile.values.aminoacid().c_str(), 8.0, -0.2f); + return new ReducedMatrix(sMat.probMatrix, sMat.subMatrixPseudoCounts, sMat.aa2num, sMat.num2aa, + sMat.alphabetSize, par.alphabetSize.values.aminoacid(), 2.0); +} + +// K-mers selected per sequence, by the same rule fillKmerPositionArray applies +// (kmersPerSequence - 1 + scale * L), plus the identity k-mer every sequence +// contributes. Computed from the average length rather than measured, because +// measuring would mean a pass over the database this stage does not otherwise +// need; it only sizes the shuffle, and the placement stays exact either way. +unsigned int estimateKmersPerSequence(Parameters &par, int dbType, uint64_t residues, + uint64_t entryCount) { + const float scale = Parameters::isEqualDbtype(dbType, Parameters::DBTYPE_NUCLEOTIDES) + ? par.kmersPerSequenceScale.values.nucleotide() + : par.kmersPerSequenceScale.values.aminoacid(); + const double averageLength = static_cast(residues) / static_cast(entryCount); + const double selected = static_cast(par.kmersPerSequence) - 1.0 + scale * averageLength; + return static_cast(std::max(1.0, std::ceil(selected))) + 1; +} + +// Extracts one key range into the bucket files. +// +// T is the position/length type of the staged KmerPosition, picked from the +// longest sequence exactly as stock kmermatcher picks it. It never reaches the +// bucket file, which always stores both fields as uint16. +template +void scanKeyRange(const std::string &seqDb, DBKeyType keyFrom, DBKeyType keyTo, int dbType, + Parameters &par, BaseMatrix *subMat, KmerBucketWriter &writer, + const KmerPartitioner &partitioner) { + DenseIndex::Info rangeInfo; + DBReader::Index *index = DenseIndex::loadRange(seqDb, keyFrom, keyTo, &rangeInfo); + if (rangeInfo.entryCount == 0) { + delete[] index; + return; + } + + DBReader reader(index, rangeInfo.entryCount, rangeInfo.dataSize, + static_cast(keyFrom + rangeInfo.entryCount - 1), dbType, + rangeInfo.maxSeqLen, par.threads); + reader.setDataFile(seqDb.c_str()); + reader.open(DBReader::NOSORT); + + // The k-mer array is never touched in bucketing mode -- every selected k-mer + // goes straight to a bucket -- so there is nothing to allocate for it. + if (Parameters::isEqualDbtype(dbType, Parameters::DBTYPE_NUCLEOTIDES)) { + fillKmerPositionArray( + NULL, SIZE_MAX, reader, par, subMat, true, 0, SIZE_MAX, NULL, &writer, &partitioner); + } else { + fillKmerPositionArray( + NULL, SIZE_MAX, reader, par, subMat, true, 0, SIZE_MAX, NULL, &writer, &partitioner); + } + + reader.close(); + // DBReader does not own an externally supplied index. + delete[] index; +} + +} // namespace + +int kmermatcherparallel(int argc, const char **argv, const Command &command) { + Parameters &par = Parameters::getInstance(); + setLinearFilterDefault(&par); + par.parseParameters(argc, argv, command, true, 0, MMseqsParameter::COMMAND_CLUSTLINEAR); + + const std::string seqDb = par.db1; + const std::string kmerDir = par.db2; + + if (DenseIndex::exists(seqDb) == false) { + Debug(Debug::ERROR) << "No dense index next to " << seqDb << ". The distributed stages " + << "address sequences by key without a resident index, which needs " + << "the dense keys createdbparallel produces.\n"; + EXIT(EXIT_FAILURE); + } + const DenseIndex::Info info = DenseIndex::readInfo(seqDb); + if (info.entryCount == 0) { + Debug(Debug::ERROR) << "Database " << seqDb << " is empty\n"; + EXIT(EXIT_FAILURE); + } + // The packed bucket record carries pos and seqLen in 16 bits, so a longer + // sequence would silently wrap rather than merely lose precision. + if (info.maxSeqLen > USHRT_MAX) { + Debug(Debug::ERROR) << "Longest sequence is " << info.maxSeqLen + << " residues; the packed k-mer record holds positions up to " + << USHRT_MAX << ".\n"; + EXIT(EXIT_FAILURE); + } + if (par.adjustKmerLength) { + // The adjusted length is a property of the whole database, and a worker + // scanning one key range cannot arrive at the value the others will. + Debug(Debug::ERROR) << "--adjust-kmer-len is not supported by the distributed map\n"; + EXIT(EXIT_FAILURE); + } + + const int dbType = FileUtil::parseDbType(seqDb.c_str()); + // Every entry stores its residues plus a newline and DBWriter's terminating + // null, so the residue count follows from the data size without a scan. + const uint64_t residues = info.dataSize - 2 * info.entryCount; + setKmerLengthAndAlphabet(par, residues, dbType); + par.printParameters(command.cmd, argc, argv, *command.params); + Debug(Debug::INFO) << "Database size: " << info.entryCount << " sequences, " << residues + << " residues\n"; + + const unsigned int kmersPerSequence = + estimateKmersPerSequence(par, dbType, residues, info.entryCount); + // The sequence database shares the scratch budget with the k-mer buckets and + // outlives them, so it is not space the shuffle can use. + const KmerShuffleSizing sizing = + deriveKmerShuffleSizing(info.entryCount, kmersPerSequence, par.scratchBudget, info.dataSize, + Util::computeMemory(par.splitMemoryLimit)); + Debug(Debug::INFO) << "K-mer shuffle: " << sizing.partitionCount << " partitions, " + << sizing.waveCount << " wave(s), " << sizing.totalKmerBytes + << " k-mer bytes, " << sizing.bytesPerPartition << " bytes per partition\n"; + if (sizing.waveCount > 1) { + Debug(Debug::ERROR) << "The scratch budget of " << par.scratchBudget << " bytes does not " + << "hold the whole k-mer shuffle; it would need " << sizing.waveCount + << " waves, each reduced and deleted before the next is extracted. " + << "Waves are not implemented yet -- raise --scratch-budget.\n"; + EXIT(EXIT_FAILURE); + } + + if (FileUtil::directoryExists(kmerDir.c_str()) == false) { + FileUtil::makeDir(kmerDir.c_str()); + } + // Derived rather than passed, so every worker runs a byte-identical command line. + const std::string coordDir = kmerDir + "/coord"; + if (FileUtil::directoryExists(coordDir.c_str()) == false) { + FileUtil::makeDir(coordDir.c_str()); + } + + ShuffleManifest manifest; + manifest.entryCount = info.entryCount; + manifest.partitionCount = sizing.partitionCount; + manifest.waveCount = sizing.waveCount; + manifest.kmerSize = static_cast(par.kmerSize); + const std::string manifestPath = coordDir + "/shuffle.info"; + { + FileLock manifestLock(coordDir + "/shuffle.lock"); + manifestLock.lock(); + if (FileUtil::fileExists(manifestPath.c_str()) == true) { + manifest.requireMatches(ShuffleManifest::read(manifestPath), manifestPath); + } else { + KmerBucketWriter::createLayout(kmerDir, sizing.partitionCount); + manifest.write(manifestPath); + } + manifestLock.unlock(); + } + + SharedCounter workerCounter(coordDir + "/worker.counter"); + const int64_t workerId = workerCounter.fetchAdd(); + const uint64_t sequencesPerItem = deriveSequencesPerItem(info.entryCount); + const int64_t itemCount = + static_cast((info.entryCount + sequencesPerItem - 1) / sequencesPerItem); + Debug(Debug::INFO) << "Worker " << workerId << " joined, " << itemCount << " key ranges of " + << sequencesPerItem << " sequences\n"; + + BaseMatrix *subMat = createSubstitutionMatrix(par, dbType); + KmerPartitioner partitioner(sizing.partitionCount); + // One writer for the whole process, shared by every thread and kept open + // across items so bucket files are appended to rather than reopened. + KmerBucketWriter writer(kmerDir, sizing.partitionCount, "w" + SSTR(workerId)); + + { + WorkQueue queue(coordDir + "/scan.queue", itemCount); + // Claimed one item at a time by the process rather than by each thread: + // the extraction inside an item is already threaded, and nesting a second + // parallel region inside a claiming one would oversubscribe the node. + const bool finished = queue.drain(workerId, [&](size_t item) { + const DBKeyType keyFrom = static_cast(item * sequencesPerItem); + const DBKeyType keyTo = static_cast( + std::min((item + 1) * sequencesPerItem, info.entryCount)); + if (info.maxSeqLen < SHRT_MAX) { + scanKeyRange(seqDb, keyFrom, keyTo, dbType, par, subMat, writer, partitioner); + } else { + scanKeyRange(seqDb, keyFrom, keyTo, dbType, par, subMat, writer, partitioner); + } + // Before drain() records the item as done, so a worker that dies can + // never leave an item marked complete whose k-mers were still buffered. + // One flush per item keeps the writes large: item size and P grow + // together, so a partition takes ~0.5 MB per flush at the 100B target. + writer.flushAll(); + }); + if (finished == false) { + Debug(Debug::ERROR) << "Map stage stalled: work remains but no item is claimable\n"; + EXIT(EXIT_FAILURE); + } + } + + writer.close(); + Debug(Debug::INFO) << "Worker " << workerId << " wrote " << writer.getRecordCount() + << " k-mers\n"; + delete subMat; + + return EXIT_SUCCESS; +} diff --git a/src/test/TestKmerPartition.cpp b/src/test/TestKmerPartition.cpp index f2b886342..2c9386ca6 100644 --- a/src/test/TestKmerPartition.cpp +++ b/src/test/TestKmerPartition.cpp @@ -95,7 +95,8 @@ static void testPartitioner() { // The plan sizes P = 8192 over the 16-bit hash space, i.e. 8 hash values per // partition. Verify that is exactly what happens, and that the whole space - // maps inside range. + // maps inside range. The 8 values a partition owns are strided rather than + // contiguous, which is the point -- see the prefix test below. std::set seen; bool inRange = true; std::map perPartition; @@ -120,6 +121,27 @@ static void testPartitioner() { allZero = allZero && single.partitionOf(static_cast(score)) == 0; } check(allZero, "a single partition collects the whole hash space"); + + // The property the balance depends on, and the one an earlier high-bits + // partitioner failed. Linclust keeps the bottom-scoring k-mers of a sequence, + // so the scores that reach a bucket are a *prefix* of the space, not a sample + // of all of it. A partitioner that spreads the whole space evenly can still + // send an entire prefix to one partition; this checks the prefix itself + // spreads. Measured on real data the high-bits version gave partition 0 68% of + // 19.9M k-mers at P = 16. + KmerPartitioner small(16); + std::map prefixCounts; + const unsigned int prefixEnd = 4096; // a plausible per-sequence threshold + for (unsigned int score = 0; score < prefixEnd; score++) { + prefixCounts[small.partitionOf(static_cast(score))]++; + } + check(prefixCounts.size() == 16, "a prefix of the score space reaches every partition"); + bool prefixBalanced = true; + for (std::map::const_iterator it = prefixCounts.begin(); + it != prefixCounts.end(); ++it) { + prefixBalanced = prefixBalanced && it->second == static_cast(prefixEnd) / 16; + } + check(prefixBalanced, "a prefix of the score space spreads evenly over the partitions"); } // The load-bearing test: run a repetitive corpus through the writer, read it @@ -262,11 +284,64 @@ static void testKmerPositionConversion() { check(sameBytes, "KmerPosition converts back into an identical record"); } +// Pins the derivation to the two scales the project actually targets, because +// the whole argument for computing P rather than exposing it is that the right +// answer differs between them. +static void testShuffleSizing() { + const uint64_t TB = 1024ULL * 1024 * 1024 * 1024; + const uint64_t GB = 1024ULL * 1024 * 1024; + + // 100B: 1e11 sequences must fit a hard 100 TB budget. Persistent load is the + // 25 TB database plus ~7.6 TB of surviving edges. + KmerShuffleSizing small = deriveKmerShuffleSizing(100000000000ULL, 21, 100 * TB, + 33 * TB, 64 * GB); + check(small.totalKmerBytes == 100000000000ULL * 21 * 24, + "100B k-mer volume follows from 21 records of 24 bytes per sequence"); + check(small.waveCount == 1, "100B fits the 100 TB budget in a single wave"); + check(small.bytesPerWave <= 100 * TB - 33 * TB, + "100B peak k-mer bytes stay inside the budget after the database and edges"); + check(small.partitionCount == 1024, "100B derives P = 1024"); + check(small.bytesPerPartition <= 64 * GB, "100B buckets fit the per-worker memory"); + + // 1T: 1e12 sequences, no hard ceiling, so only per-worker memory sets P. + KmerShuffleSizing large = deriveKmerShuffleSizing(1000000000000ULL, 21, 0, 0, 64 * GB); + check(large.waveCount == 1, "an unlimited budget means a single wave"); + check(large.partitionCount == 8192, "1T derives P = 8192"); + check(large.bytesPerPartition <= 64 * GB, "1T buckets fit the per-worker memory"); + check(large.partitionCount > small.partitionCount, + "the two target scales genuinely want different P, which is why it is derived"); + + // A budget that cannot hold the whole shuffle at once must split into waves, + // and more waves must mean smaller buckets rather than a larger P. + KmerShuffleSizing waved = deriveKmerShuffleSizing(1000000000000ULL, 21, 400 * TB, + 100 * TB, 64 * GB); + check(waved.waveCount > 1, "a budget below the k-mer volume forces multiple waves"); + check(waved.bytesPerWave <= 300 * TB, "each wave stays inside the budget"); + check(waved.bytesPerPartition <= 64 * GB, "waved buckets still fit the per-worker memory"); + check(waved.partitionCount <= large.partitionCount, + "waves reduce the peak, so no more partitions are needed than without them"); + + // P must always be a usable power of two. + bool powerOfTwo = true; + for (uint64_t seqs = 1000000; seqs <= 100000000000ULL; seqs *= 10) { + const KmerShuffleSizing s = deriveKmerShuffleSizing(seqs, 21, 0, 0, 8 * GB); + powerOfTwo = powerOfTwo && s.partitionCount > 0 && + (s.partitionCount & (s.partitionCount - 1)) == 0 && + s.partitionCount <= 65536; + } + check(powerOfTwo, "derived P is always a power of two inside the 16-bit hash space"); + + KmerShuffleSizing tiny = deriveKmerShuffleSizing(1000, 21, 0, 0, 64 * GB); + check(tiny.partitionCount == 1 && tiny.waveCount == 1, + "a small input collapses to one partition and one wave"); +} + int main(int, char **) { const std::string dir = makeTempDir(); testRecordLayout(); testPartitioner(); + testShuffleSizing(); testKmerPositionConversion(); testLosslessRoundTrip(dir); diff --git a/src/util/createdbparallel.cpp b/src/util/createdbparallel.cpp index e62fdc8f8..09ea86445 100644 --- a/src/util/createdbparallel.cpp +++ b/src/util/createdbparallel.cpp @@ -400,17 +400,21 @@ void allocateFile(const std::string &path, size_t size) { // every process numbers its threads from zero. template void runQueue(WorkQueue &queue, int threads, int64_t workerId, Body body) { + bool stalled = false; #pragma omp parallel num_threads(threads) { - while (true) { - const int64_t item = queue.claim(workerId); - if (item < 0) { - break; - } - body(static_cast(item)); - queue.complete(item, workerId); + // drain() rather than a plain claim loop: it goes back to claiming after + // waiting, so items abandoned by a crashed worker are picked up once their + // leases expire instead of being waited on by everyone forever. + if (queue.drain(workerId, body) == false) { +#pragma omp critical + stalled = true; } } + if (stalled) { + Debug(Debug::ERROR) << "Work queue stalled: work remains but no item is claimable\n"; + EXIT(EXIT_FAILURE); + } } } // namespace @@ -464,10 +468,7 @@ int createdbparallel(int argc, const char **argv, const Command &command) { chunks[chunkIdx], chunkIdx); histogram.write(path); }); - if (scanQueue.awaitAll() == false) { - Debug(Debug::ERROR) << "Scan pass stalled: work remains but no item is claimable\n"; - EXIT(EXIT_FAILURE); - } + } Debug(Debug::INFO) << "Scan pass done\n"; @@ -543,11 +544,6 @@ int createdbparallel(int argc, const char **argv, const Command &command) { emitChunk(filenames[chunks[chunkIdx].fileIdx], chunks[chunkIdx], plan, seqFd, hdrFd, seqIdxFd, hdrIdxFd); }); - if (emitQueue.awaitAll() == false) { - Debug(Debug::ERROR) << "Emit pass stalled: work remains but no item is claimable\n"; - EXIT(EXIT_FAILURE); - } - // fsync before the sentinel, so a worker that finalises after a crash // cannot read a partially flushed database. fsync(seqFd); From ec7c1393b31674838bf5a94fd7cff905101db1d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Wed, 29 Jul 2026 14:42:04 +0000 Subject: [PATCH 03/27] Add distributed reduce. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Björn Buschkämper --- src/CommandDeclarations.h | 1 + src/MMseqsBase.cpp | 15 +- src/commons/Parameters.cpp | 15 + src/commons/Parameters.h | 1 + src/linclust/CMakeLists.txt | 2 + src/linclust/CandidateEdge.cpp | 75 +++++ src/linclust/CandidateEdge.h | 81 ++++++ src/linclust/kmermatcher.cpp | 6 + src/linclust/kmermatcher.h | 20 ++ src/linclust/kmerreduceparallel.cpp | 412 ++++++++++++++++++++++++++++ 10 files changed, 627 insertions(+), 1 deletion(-) create mode 100644 src/linclust/CandidateEdge.cpp create mode 100644 src/linclust/CandidateEdge.h create mode 100644 src/linclust/kmerreduceparallel.cpp diff --git a/src/CommandDeclarations.h b/src/CommandDeclarations.h index 195d2a122..4ce863a08 100644 --- a/src/CommandDeclarations.h +++ b/src/CommandDeclarations.h @@ -28,6 +28,7 @@ extern int convertprofiledb(int argc, const char **argv, const Command& command) extern int createdb(int argc, const char **argv, const Command& command); extern int createdbparallel(int argc, const char **argv, const Command& command); extern int kmermatcherparallel(int argc, const char **argv, const Command& command); +extern int kmerreduceparallel(int argc, const char **argv, const Command& command); extern int makepaddedseqdb(int argc, const char **argv, const Command& command); extern int createindex(int argc, const char **argv, const Command& command); extern int createlinindex(int argc, const char **argv, const Command& command); diff --git a/src/MMseqsBase.cpp b/src/MMseqsBase.cpp index e336d1eb8..da9f41a1b 100644 --- a/src/MMseqsBase.cpp +++ b/src/MMseqsBase.cpp @@ -690,7 +690,20 @@ std::vector baseCommands = { "Martin Steinegger ", " ", CITATION_MMSEQS2, {{"sequenceDB", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::sequenceDb }, - {"kmerDir", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::flatfile }}}, + {"kmerDir", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::directory }}}, + {"kmerreduceparallel", kmerreduceparallel, &par.kmerreduceparallel, COMMAND_PREFILTER, + "Group shuffled k-mer partitions into candidate edges with many workers", + "# Read back the partitions kmermatcherparallel wrote, group each one, and\n" + "# write its (representative, member) candidate edges as packed binary.\n" + "# Workers claim whole partitions from /coord.\n" + "mmseqs kmerreduceparallel sequenceDB kmerDir edgeDir\n\n" + "# Run it from several nodes against the same shared filesystem\n" + "srun -N 8 mmseqs kmerreduceparallel sequenceDB kmerDir edgeDir\n", + "Martin Steinegger ", + " ", + CITATION_MMSEQS2, {{"sequenceDB", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::sequenceDb }, + {"kmerDir", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::directory }, + {"edgeDir", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::directory }}}, {"kmersearch", kmersearch, &par.kmersearch, COMMAND_PREFILTER, "Find bottom-m-hashed k-mer matches between target and query DB", NULL, diff --git a/src/commons/Parameters.cpp b/src/commons/Parameters.cpp index 64f12c8e0..f24ca517d 100644 --- a/src/commons/Parameters.cpp +++ b/src/commons/Parameters.cpp @@ -947,6 +947,21 @@ Parameters::Parameters(): kmermatcherparallel.push_back(&PARAM_COMPRESSED); kmermatcherparallel.push_back(&PARAM_V); + // kmerreduceparallel + // Grouping knobs. No k-mer extraction parameters: k and the partitioning are + // fixed by the shuffle manifest the map wrote, not re-derived here, so passing + // them would only create a way to disagree with what is on disk. + kmerreduceparallel.push_back(&PARAM_SUB_MAT); + kmerreduceparallel.push_back(&PARAM_ALPH_SIZE); + kmerreduceparallel.push_back(&PARAM_C); + kmerreduceparallel.push_back(&PARAM_COV_MODE); + kmerreduceparallel.push_back(&PARAM_INCLUDE_ONLY_EXTENDABLE); + kmerreduceparallel.push_back(&PARAM_INCLUDE_ADJACENCY); + kmerreduceparallel.push_back(&PARAM_NUM_ADJACENCY); + kmerreduceparallel.push_back(&PARAM_THREADS); + kmerreduceparallel.push_back(&PARAM_COMPRESSED); + kmerreduceparallel.push_back(&PARAM_V); + // makepaddedseqdb makepaddedseqdb.push_back(&PARAM_SUB_MAT); makepaddedseqdb.push_back(&PARAM_SCORE_BIAS); diff --git a/src/commons/Parameters.h b/src/commons/Parameters.h index 63d255f93..ab7a4698e 100644 --- a/src/commons/Parameters.h +++ b/src/commons/Parameters.h @@ -1238,6 +1238,7 @@ class Parameters { std::vector createdb; std::vector createdbparallel; std::vector kmermatcherparallel; + std::vector kmerreduceparallel; std::vector makepaddedseqdb; std::vector convert2fasta; std::vector result2flat; diff --git a/src/linclust/CMakeLists.txt b/src/linclust/CMakeLists.txt index 14bca716e..6c7dadff8 100644 --- a/src/linclust/CMakeLists.txt +++ b/src/linclust/CMakeLists.txt @@ -1,6 +1,8 @@ set(linclust_source_files linclust/kmermatcher.cpp linclust/kmermatcherparallel.cpp + linclust/kmerreduceparallel.cpp + linclust/CandidateEdge.cpp linclust/KmerPartition.cpp linclust/kmerindexdb.cpp linclust/kmersearch.cpp diff --git a/src/linclust/CandidateEdge.cpp b/src/linclust/CandidateEdge.cpp new file mode 100644 index 000000000..2fe5033c0 --- /dev/null +++ b/src/linclust/CandidateEdge.cpp @@ -0,0 +1,75 @@ +#include "CandidateEdge.h" + +#include "Debug.h" +#include "FileUtil.h" +#include "Util.h" + +#include +#include + +std::string EdgeWriter::partitionPath(const std::string &dir, unsigned int partition) { + return dir + "/p" + SSTR(partition) + ".edges"; +} + +EdgeWriter::EdgeWriter(const std::string &path, size_t bufferRecords) + : path(path), file(NULL), bufferRecords(bufferRecords), edgeCount(0), closed(false) { + buffer.reserve(bufferRecords); +} + +EdgeWriter::~EdgeWriter() { + close(); +} + +void EdgeWriter::flush() { + if (buffer.empty()) { + return; + } + if (file == NULL) { + file = fopen(path.c_str(), "wb"); + if (file == NULL) { + Debug(Debug::ERROR) << "Cannot open edge file " << path << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + } + if (fwrite(buffer.data(), sizeof(CandidateEdge), buffer.size(), file) != buffer.size()) { + Debug(Debug::ERROR) << "Cannot write " << buffer.size() << " edges to " << path << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + buffer.clear(); +} + +void EdgeWriter::append(const CandidateEdge &edge) { + buffer.push_back(edge); + if (buffer.size() >= bufferRecords) { + flush(); + } + edgeCount++; +} + +void EdgeWriter::close() { + // The destructor calls this too. Without the guard a second call would fall + // into the empty-file branch below and truncate what the first call wrote. + if (closed) { + return; + } + closed = true; + flush(); + if (file != NULL) { + // A partition that produced no edge still gets an empty file, so a reader + // can tell "this partition was reduced and had nothing" from "this + // partition was never reduced". + if (fclose(file) != 0) { + Debug(Debug::ERROR) << "Cannot close edge file " << path << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + file = NULL; + } else { + FILE *empty = fopen(path.c_str(), "wb"); + if (empty == NULL) { + Debug(Debug::ERROR) << "Cannot create edge file " << path << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + fclose(empty); + } +} diff --git a/src/linclust/CandidateEdge.h b/src/linclust/CandidateEdge.h new file mode 100644 index 000000000..fca6febc3 --- /dev/null +++ b/src/linclust/CandidateEdge.h @@ -0,0 +1,81 @@ +#ifndef MMSEQS_CANDIDATEEDGE_H +#define MMSEQS_CANDIDATEEDGE_H + +#include +#include +#include +#include + +// One (representative, member) pair proposed by the distributed reduce. +// +// Packed binary rather than a prefilter DB. A `pref` DB stores each hit as ASCII +// with a per-representative index entry, and at 1e11 sequences the index alone is +// per-key state no single node can hold. 15 bytes per edge is also ~4x smaller +// than the text form, which matters directly: this is the largest intermediate +// the pipeline writes. +struct __attribute__((__packed__)) CandidateEdge { + // 48-bit keys, as in KmerRecord: 2.8e14 keys, past the 1e12 target, and two + // bytes cheaper per side than a full 64-bit key. + uint8_t repBytes[6]; + uint8_t memberBytes[6]; + int16_t diagonal; + // Nucleotide strand. Stock carries it in bit 63 of the representative key, + // which a 48-bit key has no room for. + uint8_t reverseStrand; + // How many k-mers put this pair on this diagonal, saturating at 255. Stock's + // prefilter score, used downstream to rank and filter candidates. + uint8_t score; + + uint64_t getRep() const { return get(repBytes); } + uint64_t getMember() const { return get(memberBytes); } + void setRep(uint64_t key) { set(repBytes, key); } + void setMember(uint64_t key) { set(memberBytes, key); } + +private: + static uint64_t get(const uint8_t *bytes) { + uint64_t value = 0; + for (int i = 5; i >= 0; i--) { + value = (value << 8) | bytes[i]; + } + return value; + } + static void set(uint8_t *bytes, uint64_t value) { + for (int i = 0; i < 6; i++) { + bytes[i] = static_cast(value & 0xFF); + value >>= 8; + } + } +}; + +// Buffered writer for one partition's edge file. +// +// One file per partition, written whole, so a partition redone after a crash +// simply overwrites its file. That makes the reduce idempotent, unlike the map, +// whose per-worker shards can retain a dead worker's partial output. +class EdgeWriter { +public: + EdgeWriter(const std::string &path, size_t bufferRecords = 1024 * 1024); + ~EdgeWriter(); + + void append(const CandidateEdge &edge); + void close(); + + uint64_t getEdgeCount() const { return edgeCount; } + + static std::string partitionPath(const std::string &dir, unsigned int partition); + +private: + EdgeWriter(const EdgeWriter &); + EdgeWriter &operator=(const EdgeWriter &); + + void flush(); + + std::string path; + FILE *file; + std::vector buffer; + size_t bufferRecords; + uint64_t edgeCount; + bool closed; +}; + +#endif diff --git a/src/linclust/kmermatcher.cpp b/src/linclust/kmermatcher.cpp index 6c042a2fa..6e1bf06a1 100644 --- a/src/linclust/kmermatcher.cpp +++ b/src/linclust/kmermatcher.cpp @@ -827,6 +827,12 @@ template size_t assignGroup<0, int, true, false>(KmerPosition template size_t assignGroup<1, short, true, false>(KmerPosition *kmers, KmerPosition *writeSeqPair, bool includeOnlyExtendable, int covMode, float covThr, SequenceWeights *sequenceWeights, float weightThr, int threads, std::vector& threadOffsets, BaseMatrix *subMat, AssignGroupMask assignGroupMask, ComputationPhase phase, short *countTable); template size_t assignGroup<1, int, true, false>(KmerPosition *kmers, KmerPosition *writeSeqPair, bool includeOnlyExtendable, int covMode, float covThr, SequenceWeights *sequenceWeights, float weightThr, int threads, std::vector& threadOffsets, BaseMatrix *subMat, AssignGroupMask assignGroupMask, ComputationPhase phase, short *countTable); +// Distributed reduce: adjacency and inline sequence lengths both on. +template size_t assignGroup<0, short, true, true>(KmerPosition *kmers, KmerPosition *writeSeqPair, bool includeOnlyExtendable, int covMode, float covThr, SequenceWeights *sequenceWeights, float weightThr, int threads, std::vector& threadOffsets, BaseMatrix *subMat, AssignGroupMask assignGroupMask, ComputationPhase phase, short *countTable); +template size_t assignGroup<0, int, true, true>(KmerPosition *kmers, KmerPosition *writeSeqPair, bool includeOnlyExtendable, int covMode, float covThr, SequenceWeights *sequenceWeights, float weightThr, int threads, std::vector& threadOffsets, BaseMatrix *subMat, AssignGroupMask assignGroupMask, ComputationPhase phase, short *countTable); +template size_t assignGroup<1, short, true, true>(KmerPosition *kmers, KmerPosition *writeSeqPair, bool includeOnlyExtendable, int covMode, float covThr, SequenceWeights *sequenceWeights, float weightThr, int threads, std::vector& threadOffsets, BaseMatrix *subMat, AssignGroupMask assignGroupMask, ComputationPhase phase, short *countTable); +template size_t assignGroup<1, int, true, true>(KmerPosition *kmers, KmerPosition *writeSeqPair, bool includeOnlyExtendable, int covMode, float covThr, SequenceWeights *sequenceWeights, float weightThr, int threads, std::vector& threadOffsets, BaseMatrix *subMat, AssignGroupMask assignGroupMask, ComputationPhase phase, short *countTable); + template static void runIteration( AssignGroupMask mask, int &iteration, size_t &writePos, diff --git a/src/linclust/kmermatcher.h b/src/linclust/kmermatcher.h index 589e11753..b3187dad6 100644 --- a/src/linclust/kmermatcher.h +++ b/src/linclust/kmermatcher.h @@ -267,6 +267,26 @@ KmerPosition * doComputation(size_t totalKme template KmerPosition *initKmerPositionMemory(size_t size); +class SequenceWeights; + +// Greedy grouping of the sorted k-mer array into (representative, member) pairs. +// +// Declared here so the distributed reduce can call it on one k-mer partition at a +// time. That is exactly the same call stock makes on one split, and it is correct +// per partition for the same reason: grouping only ever compares equal k-mers, and +// equal k-mers always share a partition. +// +// On return the entries hold `.kmer` = representative key, `.id` = member key and +// `.pos` = diagonal, written into writeSeqPair when it is non-NULL and in place +// otherwise. +template +size_t assignGroup(KmerPosition *hashSeqPair, + KmerPosition *writeSeqPair, + bool includeOnlyExtendable, int covMode, float covThr, + SequenceWeights *sequenceWeights, float weightThr, int threads, + std::vector &threadOffsets, BaseMatrix *subMat, + AssignGroupMask assignGroupMask, ComputationPhase phase, short *countTable); + template std::pair fillKmerPositionArray(KmerPosition * kmerArray, size_t kmerArraySize, DBReader &seqDbr, Parameters & par, BaseMatrix * subMat, bool hashWholeSequence, diff --git a/src/linclust/kmerreduceparallel.cpp b/src/linclust/kmerreduceparallel.cpp new file mode 100644 index 000000000..c62f5f017 --- /dev/null +++ b/src/linclust/kmerreduceparallel.cpp @@ -0,0 +1,412 @@ +/* + * kmerreduceparallel -- the reduce half of the distributed linclust. + * + * Reads back one k-mer partition written by kmermatcherparallel, groups it, and + * writes the resulting (representative, member) candidate edges. Many worker + * processes run the same command line and claim partitions from a shared work + * queue, exactly as the map stage claims key ranges. + * + * Why grouping a partition in isolation gives the same answer as grouping the + * whole database: the greedy only ever compares *equal* k-mers, and the partition + * of a k-mer is a pure function of the k-mer (see KmerPartition.h), so every + * occurrence of a k-mer is in the partition being read and nowhere else. The + * grouping call is stock's `assignGroup`, unchanged -- this command supplies it + * with one partition where stock supplies it with one split. + * + * Two things here exist because the stock reduce cannot run at 1e12 sequences: + * - lengths come from the k-mer record, not from `seqkey_to_len[dbKeySize]` + * (kmermatcher.cpp:1236), which is sized by key space; + * - output is packed binary edges rather than a prefilter DB, whose per-key + * index is likewise per-key state no node can hold. + * Neither the sequence database nor any per-key table is opened at all: a + * partition is self-contained, which is the whole point of carrying seqLen and + * the adjacent residues in the record. + */ +#include "Command.h" +#include "CandidateEdge.h" +#include "Debug.h" +#include "DenseIndex.h" +#include "FastSort.h" +#include "FileUtil.h" +#include "KmerPartition.h" +#include "NucleotideMatrix.h" +#include "ParallelCoordination.h" +#include "Parameters.h" +#include "ReducedMatrix.h" +#include "SubstitutionMatrix.h" +#include "Util.h" +#include "kmermatcher.h" + +#include +#include +#include +#include +#include + +namespace { + +BaseMatrix *createSubstitutionMatrix(Parameters &par, int dbType) { + if (Parameters::isEqualDbtype(dbType, Parameters::DBTYPE_NUCLEOTIDES)) { + return new NucleotideMatrix(par.scoringMatrixFile.values.nucleotide().c_str(), 1.0, 0.0); + } + if (par.alphabetSize.values.aminoacid() == 21) { + return new SubstitutionMatrix(par.scoringMatrixFile.values.aminoacid().c_str(), 2.0, 0.0); + } + SubstitutionMatrix sMat(par.scoringMatrixFile.values.aminoacid().c_str(), 8.0, -0.2f); + return new ReducedMatrix(sMat.probMatrix, sMat.subMatrixPseudoCounts, sMat.aa2num, sMat.num2aa, + sMat.alphabetSize, par.alphabetSize.values.aminoacid(), 2.0); +} + +// Streams a partition's shards straight into the KmerPosition array. +// +// Reading into a vector first and converting afterwards would hold +// both representations at once, which is 24 extra bytes per k-mer at the moment +// of peak memory -- ~50% more for a stage whose partition size is chosen to just +// fit a node. +template +size_t readPartitionAsPositions(const std::string &kmerDir, unsigned int partition, + KmerPosition *out, size_t capacity) { + const std::vector shards = KmerBucketReader::shardFiles(kmerDir, partition); + const size_t blockRecords = 1024 * 1024; + std::vector block(blockRecords); + size_t filled = 0; + for (size_t i = 0; i < shards.size(); i++) { + FILE *file = FileUtil::openFileOrDie(shards[i].c_str(), "rb", true); + while (true) { + const size_t got = fread(block.data(), sizeof(KmerRecord), blockRecords, file); + if (got == 0) { + break; + } + if (filled + got > capacity) { + Debug(Debug::ERROR) << "Partition " << partition << " holds more k-mers than the " + << "shard sizes reported. A shard was written while this was " + << "reading it.\n"; + EXIT(EXIT_FAILURE); + } + for (size_t r = 0; r < got; r++) { + kmerRecordToPosition(block[r], out[filled + r]); + } + filled += got; + } + fclose(file); + } + return filled; +} + +// Drops records that are byte-identical to their predecessor. +// +// The map is not idempotent across a crash: a worker killed mid-item may have +// flushed part of it, and the worker that redoes the item writes the same records +// into a different shard. Removing them here is exact rather than a heuristic -- +// (kmer, id, pos) names one k-mer occurrence in one sequence, so identical +// records can only be the same occurrence written twice. Duplicates are adjacent +// after the sort because every field they are ordered by is equal. +template +size_t dropDuplicates(KmerPosition *positions, size_t count) { + if (count == 0) { + return 0; + } + size_t out = 1; + for (size_t i = 1; i < count; i++) { + if (memcmp(&positions[i], &positions[out - 1], sizeof(positions[0])) != 0) { + positions[out] = positions[i]; + out++; + } + } + return out; +} + +// Splits the sorted array into per-thread ranges that never cut a k-mer group, +// so each thread's greedy sees whole groups. Same rule as doComputation +// (kmermatcher.cpp:1006-1047). +template +void buildThreadOffsets(KmerPosition *positions, size_t count, int threads, + bool isNucleotide, std::vector &threadOffsets) { + threadOffsets.clear(); + threadOffsets.push_back(0); + const size_t splitSize = count / threads; + for (int thread = 1; thread < threads; thread++) { + size_t prevKmer = positions[thread * splitSize].kmer; + if (prevKmer == SIZE_MAX) { + for (int i = thread; i < threads; i++) { + threadOffsets.push_back(count); + } + break; + } + if (isNucleotide) { + prevKmer = BIT_SET(prevKmer, 63); + } + bool wasSet = false; + for (size_t pos = thread * splitSize; pos < count; pos++) { + size_t currKmer = positions[pos].kmer; + if (isNucleotide) { + currKmer = BIT_SET(currKmer, 63); + } + if (prevKmer != currKmer) { + wasSet = true; + threadOffsets.push_back(pos); + break; + } + } + if (wasSet == false) { + for (int i = thread; i < threads; i++) { + threadOffsets.push_back(count); + } + break; + } + } + threadOffsets.push_back(count); +} + + +// Collapses one grouping round's output into (representative, member) edges. +// +// The same pair is produced once for every k-mer the two sequences share, so +// emitting them raw duplicates heavily -- measured 4.1 M records for 1.1 M +// distinct pairs on 1M sequences. Stock collapses them the same way in +// writeKmerMatcherResult (kmermatcher.cpp:1789): keep the diagonal that the most +// k-mers agree on, and use that count as the prefilter score. The array is sorted +// by (rep, member, diagonal), so both runs are contiguous. +template +void collectRoundEdges(KmerPosition *grouped, size_t writePos, bool isNucleotide, + std::vector &edges) { + CandidateEdge edge; + size_t i = 0; + while (i < writePos && grouped[i].kmer != SIZE_MAX) { + const size_t repRaw = grouped[i].kmer; + const DBKeyType member = grouped[i].id; + T bestDiagonal = grouped[i].pos; + size_t bestCount = 0; + T runDiagonal = grouped[i].pos; + size_t runCount = 0; + size_t j = i; + while (j < writePos && grouped[j].kmer == repRaw && grouped[j].id == member) { + if (grouped[j].pos == runDiagonal) { + runCount++; + } else { + runDiagonal = grouped[j].pos; + runCount = 1; + } + if (runCount > bestCount) { + bestCount = runCount; + bestDiagonal = runDiagonal; + } + j++; + } + i = j; + + size_t rep = repRaw; + // Stock keeps the strand in bit 63 of the representative key; a 48-bit key + // has no room for it, so it moves into its own field. + edge.reverseStrand = 0; + if (isNucleotide) { + edge.reverseStrand = BIT_CHECK(rep, 63) == false; + rep = BIT_CLEAR(rep, 63); + } + // A sequence being its own representative carries no information here: + // keys are dense, so any key that never appears as a member is its own + // representative by construction, and the final greedy derives singletons + // from that rather than from a marker edge. + if (static_cast(rep) == member) { + continue; + } + edge.setRep(static_cast(rep)); + edge.setMember(static_cast(member)); + edge.diagonal = static_cast(bestDiagonal); + edge.score = static_cast(std::min(bestCount, 255)); + edges.push_back(edge); + } +} + +// Orders edges so duplicates of a pair are adjacent, best score first. +bool compareEdge(const CandidateEdge &a, const CandidateEdge &b) { + const uint64_t ra = a.getRep(), rb = b.getRep(); + if (ra != rb) return ra < rb; + const uint64_t ma = a.getMember(), mb = b.getMember(); + if (ma != mb) return ma < mb; + return a.score > b.score; +} + +// Groups one partition and writes its edges. +template +uint64_t reducePartition(const std::string &kmerDir, const std::string &edgeDir, + unsigned int partition, int dbType, Parameters &par, BaseMatrix *subMat) { + const bool isNucleotide = Parameters::isEqualDbtype(dbType, Parameters::DBTYPE_NUCLEOTIDES); + EdgeWriter writer(EdgeWriter::partitionPath(edgeDir, partition)); + + const uint64_t recordCount = KmerBucketReader::countRecords(kmerDir, partition); + if (recordCount == 0) { + writer.close(); + return 0; + } + + // One slot past the end for the SIZE_T_MAX sentinel assignGroup stops on, + // matching initKmerPositionMemory (kmermatcher.cpp:43). + KmerPosition *positions = + new (std::nothrow) KmerPosition[recordCount + 1]; + Util::checkAllocation(positions, "Cannot allocate the k-mer partition"); + + size_t count = readPartitionAsPositions(kmerDir, partition, positions, recordCount); + if (isNucleotide) { + SORT_PARALLEL(positions, positions + count, + KmerPosition::compareRepSequenceAndIdAndPosReverse); + } else { + SORT_PARALLEL(positions, positions + count, + KmerPosition::compareRepSequenceAndIdAndPos); + } + count = dropDuplicates(positions, count); + memset(&positions[count], 0xFF, sizeof(positions[0])); + + std::vector threadOffsets; + buildThreadOffsets(positions, count, par.threads, isNucleotide, threadOffsets); + + // assignGroup writes the grouped pairs here rather than over its input, so a + // round can be re-run over the k-mers rather than over the previous result. + KmerPosition *grouped = + new (std::nothrow) KmerPosition[count + 1]; + Util::checkAllocation(grouped, "Cannot allocate the grouping output"); + + // linclust v2's rounds, ported. + // + // Round 0 picks the longest sequence of each k-mer group as its centre; every + // later round re-picks it by adjacency agreement. The rounds are *stateful*: + // assignGroup swaps the chosen centre to the front of its group inside the + // k-mer array (kmermatcher.cpp:607,638), so each round starts from the previous + // round's arrangement and finds pairs the earlier ones did not. Measured on + // stock, the rounds are strictly additive -- 3 rounds is a superset of 1, + // adding 90,628 pairs and losing none. + // + // That mutation is the whole reason this ports for free: all the round-to-round + // state lives in the k-mer array, which is partition-local. Nothing global is + // needed, so the rounds cost the same here as they do in stock. + const int rounds = par.includeAdjacency ? 1 + par.adjIteration : 1; + std::vector edges; + for (int round = 0; round < rounds; round++) { + const AssignGroupMask mask = + (round == 0) ? AssignGroupFeature::Default : AssignGroupFeature::AdjacentSeq; + size_t writePos = 0; + if (isNucleotide) { + writePos = assignGroup( + positions, grouped, par.includeOnlyExtendable, par.covMode, par.covThr, NULL, + par.weightThr, par.threads, threadOffsets, subMat, mask, ComputationPhase::Main, + NULL); + SORT_PARALLEL(grouped, grouped + writePos, + KmerPosition::compareRepSequenceAndIdAndDiagReverse); + } else { + writePos = assignGroup( + positions, grouped, par.includeOnlyExtendable, par.covMode, par.covThr, NULL, + par.weightThr, par.threads, threadOffsets, subMat, mask, ComputationPhase::Main, + NULL); + SORT_PARALLEL(grouped, grouped + writePos, + KmerPosition::compareRepSequenceAndIdAndDiag); + } + collectRoundEdges(grouped, writePos, isNucleotide, edges); + } + delete[] positions; + delete[] grouped; + + // Rounds overlap heavily by construction, so collapse the union here rather + // than writing the same pair once per round. Keeping the highest score means + // the round that agreed on the most k-mers wins. + SORT_PARALLEL(edges.begin(), edges.end(), compareEdge); + for (size_t i = 0; i < edges.size(); i++) { + if (i > 0 && edges[i].getRep() == edges[i - 1].getRep() && + edges[i].getMember() == edges[i - 1].getMember()) { + continue; + } + writer.append(edges[i]); + } + + writer.close(); + return writer.getEdgeCount(); +} + +} // namespace + +int kmerreduceparallel(int argc, const char **argv, const Command &command) { + Parameters &par = Parameters::getInstance(); + setLinearFilterDefault(&par); + par.parseParameters(argc, argv, command, true, 0, MMseqsParameter::COMMAND_CLUSTLINEAR); + + const std::string seqDb = par.db1; + const std::string kmerDir = par.db2; + const std::string edgeDir = par.db3; + + // Only the header is read, for the sequence type and the longest sequence. + // The partitions themselves are self-contained. + const int dbType = FileUtil::parseDbType(seqDb.c_str()); + const DenseIndex::Info info = DenseIndex::readInfo(seqDb); + + const std::string coordDir = kmerDir + "/coord"; + const std::string manifestPath = coordDir + "/shuffle.info"; + if (FileUtil::fileExists(manifestPath.c_str()) == false) { + Debug(Debug::ERROR) << "No k-mer shuffle at " << kmerDir << " (" << manifestPath + << " is missing). Run kmermatcherparallel first.\n"; + EXIT(EXIT_FAILURE); + } + unsigned int partitionCount = 0; + unsigned int kmerSize = 0; + { + FILE *file = FileUtil::openFileOrDie(manifestPath.c_str(), "r", true); + char name[64]; + size_t value; + while (fscanf(file, "%63s\t%zu\n", name, &value) == 2) { + const std::string key = name; + if (key == "partitionCount") { + partitionCount = static_cast(value); + } else if (key == "kmerSize") { + kmerSize = static_cast(value); + } + } + fclose(file); + } + if (partitionCount == 0) { + Debug(Debug::ERROR) << "Shuffle manifest " << manifestPath << " has no partition count\n"; + EXIT(EXIT_FAILURE); + } + // The map decided k, so take it from the manifest rather than re-deriving it + // here: the two must agree, and the map's value is the one on disk. + par.kmerSize = static_cast(kmerSize); + par.printParameters(command.cmd, argc, argv, *command.params); + Debug(Debug::INFO) << "Reducing " << partitionCount << " partitions of " << kmerDir << "\n"; + + if (FileUtil::directoryExists(edgeDir.c_str()) == false) { + FileUtil::makeDir(edgeDir.c_str()); + } + const std::string reduceCoordDir = edgeDir + "/coord"; + if (FileUtil::directoryExists(reduceCoordDir.c_str()) == false) { + FileUtil::makeDir(reduceCoordDir.c_str()); + } + + SharedCounter workerCounter(reduceCoordDir + "/worker.counter"); + const int64_t workerId = workerCounter.fetchAdd(); + Debug(Debug::INFO) << "Worker " << workerId << " joined\n"; + + BaseMatrix *subMat = createSubstitutionMatrix(par, dbType); + uint64_t edgeCount = 0; + { + WorkQueue queue(reduceCoordDir + "/reduce.queue", static_cast(partitionCount)); + // One partition at a time per process: the sort and the greedy inside a + // partition are already threaded, and a partition is sized to fill a node. + const bool finished = queue.drain(workerId, [&](size_t partition) { + if (info.maxSeqLen < SHRT_MAX) { + edgeCount += reducePartition(kmerDir, edgeDir, + static_cast(partition), dbType, + par, subMat); + } else { + edgeCount += reducePartition(kmerDir, edgeDir, + static_cast(partition), dbType, par, + subMat); + } + }); + if (finished == false) { + Debug(Debug::ERROR) << "Reduce stage stalled: work remains but no partition is claimable\n"; + EXIT(EXIT_FAILURE); + } + } + + Debug(Debug::INFO) << "Worker " << workerId << " wrote " << edgeCount << " candidate edges\n"; + delete subMat; + + return EXIT_SUCCESS; +} From 67fce301e232abf55ced4a322a2052e786cb5188 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Thu, 30 Jul 2026 07:58:28 +0000 Subject: [PATCH 04/27] Add distributed alignment. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Björn Buschkämper --- src/CommandDeclarations.h | 1 + src/MMseqsBase.cpp | 10 + src/alignment/Align2clust.cpp | 25 -- src/alignment/Matcher.cpp | 29 ++- src/alignment/Matcher.h | 7 + src/alignment/rescorediagonal.cpp | 24 -- src/commons/Parameters.cpp | 18 ++ src/commons/Parameters.h | 1 + src/linclust/CMakeLists.txt | 2 + src/linclust/CandidateEdge.cpp | 117 ++++++++++ src/linclust/CandidateEdge.h | 56 +++++ src/linclust/PartitionSequences.cpp | 146 ++++++++++++ src/linclust/PartitionSequences.h | 62 +++++ src/linclust/alignparallel.cpp | 347 ++++++++++++++++++++++++++++ src/linclust/kmerreduceparallel.cpp | 165 ++++++++----- 15 files changed, 901 insertions(+), 109 deletions(-) create mode 100644 src/linclust/PartitionSequences.cpp create mode 100644 src/linclust/PartitionSequences.h create mode 100644 src/linclust/alignparallel.cpp diff --git a/src/CommandDeclarations.h b/src/CommandDeclarations.h index 4ce863a08..455424e0d 100644 --- a/src/CommandDeclarations.h +++ b/src/CommandDeclarations.h @@ -29,6 +29,7 @@ extern int createdb(int argc, const char **argv, const Command& command); extern int createdbparallel(int argc, const char **argv, const Command& command); extern int kmermatcherparallel(int argc, const char **argv, const Command& command); extern int kmerreduceparallel(int argc, const char **argv, const Command& command); +extern int alignparallel(int argc, const char **argv, const Command& command); extern int makepaddedseqdb(int argc, const char **argv, const Command& command); extern int createindex(int argc, const char **argv, const Command& command); extern int createlinindex(int argc, const char **argv, const Command& command); diff --git a/src/MMseqsBase.cpp b/src/MMseqsBase.cpp index da9f41a1b..905148de7 100644 --- a/src/MMseqsBase.cpp +++ b/src/MMseqsBase.cpp @@ -704,6 +704,16 @@ std::vector baseCommands = { CITATION_MMSEQS2, {{"sequenceDB", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::sequenceDb }, {"kmerDir", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::directory }, {"edgeDir", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::directory }}}, + {"alignparallel", alignparallel, &par.alignparallel, COMMAND_ALIGNMENT, + "Align candidate edges bucketed by representative key, with many workers", + "# Merge the duplicate copies the k-mer partitions produced, align each pair\n" + "# once, and write the survivors. Workers claim representative-key buckets.\n" + "mmseqs alignparallel sequenceDB edgeDir alnDir --min-seq-id 0.9 -c 0.8\n", + "Martin Steinegger ", + " ", + CITATION_MMSEQS2, {{"sequenceDB", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::sequenceDb }, + {"edgeDir", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::directory }, + {"alnDir", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::directory }}}, {"kmersearch", kmersearch, &par.kmersearch, COMMAND_PREFILTER, "Find bottom-m-hashed k-mer matches between target and query DB", NULL, diff --git a/src/alignment/Align2clust.cpp b/src/alignment/Align2clust.cpp index d6308dc18..44bf3f4de 100644 --- a/src/alignment/Align2clust.cpp +++ b/src/alignment/Align2clust.cpp @@ -203,31 +203,6 @@ static void pushClusterResult(ClusterResult &&clusterResult) { } } -static float parsePrecisionLib(const std::string &scoreFile, double targetSeqid, double targetCov, double targetPrecision) { - std::stringstream in(scoreFile); - std::string line; - int intTargetSeqid = static_cast((targetSeqid + 0.0001) * 100); - int seqIdRest = (intTargetSeqid % 5); - targetSeqid = static_cast(intTargetSeqid - seqIdRest) / 100; - targetCov = static_cast(static_cast((targetCov + 0.0001) * 10)) / 10; - - while (std::getline(in, line)) { - std::vector values = Util::split(line, " "); - float cov = strtod(values[0].c_str(), NULL); - float seqid = strtod(values[1].c_str(), NULL); - float scorePerCol = strtod(values[2].c_str(), NULL); - float precision = strtod(values[3].c_str(), NULL); - if (MathUtil::AreSame(cov, targetCov) && MathUtil::AreSame(seqid, targetSeqid) && precision >= targetPrecision) { - return scorePerCol; - } - } - - Debug(Debug::WARNING) << "Can not find any score per column for coverage " - << targetCov << " and sequence identity " << targetSeqid - << ". No hit will be filtered.\n"; - return 0; -} - static void writeClustering(DBWriter *dbWriter, const std::pair * results, size_t dbSize) { std::string resultString; resultString.reserve(1024 * 1024 * 1024); diff --git a/src/alignment/Matcher.cpp b/src/alignment/Matcher.cpp index d054a3508..a90519537 100644 --- a/src/alignment/Matcher.cpp +++ b/src/alignment/Matcher.cpp @@ -1,6 +1,8 @@ #include #include #include "Matcher.h" +#include +#include "MathUtil.h" #include "Util.h" #include "Parameters.h" #include "StripedSmithWaterman.h" @@ -411,4 +413,29 @@ std::string getCovSeqidQscPercMinDiagTargetCov() { reinterpret_cast(CovSeqidQscPercMinDiagTargetCov_lib), CovSeqidQscPercMinDiagTargetCov_lib_len ); -} \ No newline at end of file +} + +float parsePrecisionLib(const std::string &scoreFile, double targetSeqid, double targetCov, double targetPrecision) { + std::stringstream in(scoreFile); + std::string line; + int intTargetSeqid = static_cast((targetSeqid + 0.0001) * 100); + int seqIdRest = (intTargetSeqid % 5); + targetSeqid = static_cast(intTargetSeqid - seqIdRest) / 100; + targetCov = static_cast(static_cast((targetCov + 0.0001) * 10)) / 10; + + while (std::getline(in, line)) { + std::vector values = Util::split(line, " "); + float cov = strtod(values[0].c_str(), NULL); + float seqid = strtod(values[1].c_str(), NULL); + float scorePerCol = strtod(values[2].c_str(), NULL); + float precision = strtod(values[3].c_str(), NULL); + if (MathUtil::AreSame(cov, targetCov) && MathUtil::AreSame(seqid, targetSeqid) && precision >= targetPrecision) { + return scorePerCol; + } + } + + Debug(Debug::WARNING) << "Can not find any score per column for coverage " + << targetCov << " and sequence identity " << targetSeqid + << ". No hit will be filtered.\n"; + return 0; +} diff --git a/src/alignment/Matcher.h b/src/alignment/Matcher.h index 3856102dd..c3ac6b942 100644 --- a/src/alignment/Matcher.h +++ b/src/alignment/Matcher.h @@ -255,6 +255,13 @@ class Matcher{ }; std::string getCovSeqidQscPercMinDiag(); + +// Looks up the score-per-column cutoff that reaches targetPrecision at the given +// coverage and sequence identity, from one of the precision libraries above. +// Lives here rather than in a caller because more than one stage needs it and the +// libraries it parses are here. +float parsePrecisionLib(const std::string &scoreFile, double targetSeqid, double targetCov, + double targetPrecision); std::string getCovSeqidQscPercMinDiagTargetCov(); #endif diff --git a/src/alignment/rescorediagonal.cpp b/src/alignment/rescorediagonal.cpp index 3b53245d3..9a234ed26 100644 --- a/src/alignment/rescorediagonal.cpp +++ b/src/alignment/rescorediagonal.cpp @@ -17,30 +17,6 @@ #include #endif -float parsePrecisionLib(const std::string &scoreFile, double targetSeqid, double targetCov, double targetPrecision) { - std::stringstream in(scoreFile); - std::string line; - // find closest lower seq. id in a grid of size 5 - int intTargetSeqid = static_cast((targetSeqid + 0.0001) * 100); - int seqIdRest = (intTargetSeqid % 5); - targetSeqid = static_cast(intTargetSeqid - seqIdRest) / 100; - // find closest lower cov. id in a grid of size 10 - targetCov = static_cast(static_cast((targetCov + 0.0001) * 10)) / 10; - while (std::getline(in, line)) { - std::vector values = Util::split(line, " "); - float cov = strtod(values[0].c_str(), NULL); - float seqid = strtod(values[1].c_str(), NULL); - float scorePerCol = strtod(values[2].c_str(), NULL); - float precision = strtod(values[3].c_str(), NULL); - if (MathUtil::AreSame(cov, targetCov) && MathUtil::AreSame(seqid, targetSeqid) && precision >= targetPrecision) { - return scorePerCol; - } - } - Debug(Debug::WARNING) << "Can not find any score per column for coverage " - << targetCov << " and sequence identity " << targetSeqid << ". No hit will be filtered.\n"; - - return 0; -} int doRescorediagonal(Parameters &par, DBWriter &resultWriter, diff --git a/src/commons/Parameters.cpp b/src/commons/Parameters.cpp index f24ca517d..538c01edf 100644 --- a/src/commons/Parameters.cpp +++ b/src/commons/Parameters.cpp @@ -956,12 +956,30 @@ Parameters::Parameters(): kmerreduceparallel.push_back(&PARAM_C); kmerreduceparallel.push_back(&PARAM_COV_MODE); kmerreduceparallel.push_back(&PARAM_INCLUDE_ONLY_EXTENDABLE); + kmerreduceparallel.push_back(&PARAM_SPLIT_MEMORY_LIMIT); kmerreduceparallel.push_back(&PARAM_INCLUDE_ADJACENCY); kmerreduceparallel.push_back(&PARAM_NUM_ADJACENCY); kmerreduceparallel.push_back(&PARAM_THREADS); kmerreduceparallel.push_back(&PARAM_COMPRESSED); kmerreduceparallel.push_back(&PARAM_V); + // alignparallel + // Alignment knobs only: the edge buckets and the key ranges are fixed by the + // manifest kmerreduceparallel wrote, not re-derived here. + alignparallel.push_back(&PARAM_SUB_MAT); + alignparallel.push_back(&PARAM_E); + alignparallel.push_back(&PARAM_C); + alignparallel.push_back(&PARAM_COV_MODE); + alignparallel.push_back(&PARAM_MIN_SEQ_ID); + alignparallel.push_back(&PARAM_MIN_ALN_LEN); + alignparallel.push_back(&PARAM_SEQ_ID_MODE); + alignparallel.push_back(&PARAM_GAP_OPEN); + alignparallel.push_back(&PARAM_GAP_EXTEND); + alignparallel.push_back(&PARAM_NO_COMP_BIAS_CORR); + alignparallel.push_back(&PARAM_THREADS); + alignparallel.push_back(&PARAM_COMPRESSED); + alignparallel.push_back(&PARAM_V); + // makepaddedseqdb makepaddedseqdb.push_back(&PARAM_SUB_MAT); makepaddedseqdb.push_back(&PARAM_SCORE_BIAS); diff --git a/src/commons/Parameters.h b/src/commons/Parameters.h index ab7a4698e..a40ab9014 100644 --- a/src/commons/Parameters.h +++ b/src/commons/Parameters.h @@ -1239,6 +1239,7 @@ class Parameters { std::vector createdbparallel; std::vector kmermatcherparallel; std::vector kmerreduceparallel; + std::vector alignparallel; std::vector makepaddedseqdb; std::vector convert2fasta; std::vector result2flat; diff --git a/src/linclust/CMakeLists.txt b/src/linclust/CMakeLists.txt index 6c7dadff8..d45f24626 100644 --- a/src/linclust/CMakeLists.txt +++ b/src/linclust/CMakeLists.txt @@ -2,7 +2,9 @@ set(linclust_source_files linclust/kmermatcher.cpp linclust/kmermatcherparallel.cpp linclust/kmerreduceparallel.cpp + linclust/alignparallel.cpp linclust/CandidateEdge.cpp + linclust/PartitionSequences.cpp linclust/KmerPartition.cpp linclust/kmerindexdb.cpp linclust/kmersearch.cpp diff --git a/src/linclust/CandidateEdge.cpp b/src/linclust/CandidateEdge.cpp index 2fe5033c0..8abf41713 100644 --- a/src/linclust/CandidateEdge.cpp +++ b/src/linclust/CandidateEdge.cpp @@ -5,8 +5,12 @@ #include "Util.h" #include +#include #include +#include +#include + std::string EdgeWriter::partitionPath(const std::string &dir, unsigned int partition) { return dir + "/p" + SSTR(partition) + ".edges"; } @@ -73,3 +77,116 @@ void EdgeWriter::close() { fclose(empty); } } + +std::string EdgeBucketWriter::bucketDir(const std::string &dir, unsigned int bucket) { + return dir + "/r" + SSTR(bucket); +} + +void EdgeBucketWriter::createLayout(const std::string &dir, unsigned int bucketCount) { + if (FileUtil::directoryExists(dir.c_str()) == false) { + FileUtil::makeDir(dir.c_str()); + } + for (unsigned int b = 0; b < bucketCount; b++) { + const std::string path = bucketDir(dir, b); + if (FileUtil::directoryExists(path.c_str()) == false) { + // Racing workers may both try; only a failure that also leaves no + // directory behind is real. + if (mkdir(path.c_str(), 0777) != 0 && FileUtil::directoryExists(path.c_str()) == false) { + Debug(Debug::ERROR) << "Cannot create edge bucket " << path << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + } + } +} + +EdgeBucketWriter::EdgeBucketWriter(const std::string &dir, unsigned int bucketCount, + const std::string &shardId, size_t bufferBudgetBytes) + : dir(dir), shardId(shardId), bucketCount(bucketCount), edgeCount(0), closed(false) { + const size_t perBucket = bufferBudgetBytes / (bucketCount * sizeof(CandidateEdge)); + edgesPerBuffer = std::max(perBucket, 64); + buffers.resize(bucketCount); + files.assign(bucketCount, NULL); +} + +EdgeBucketWriter::~EdgeBucketWriter() { + close(); +} + +void EdgeBucketWriter::flush(unsigned int bucket) { + std::vector &buffer = buffers[bucket]; + if (buffer.empty()) { + return; + } + if (files[bucket] == NULL) { + // Opened lazily: a worker whose partitions produced nothing for a bucket + // should not cost a descriptor or an empty file. + const std::string path = bucketDir(dir, bucket) + "/" + shardId + ".edges"; + files[bucket] = fopen(path.c_str(), "wb"); + if (files[bucket] == NULL) { + Debug(Debug::ERROR) << "Cannot open edge bucket " << path << ": " << strerror(errno) + << "\n"; + EXIT(EXIT_FAILURE); + } + } + if (fwrite(buffer.data(), sizeof(CandidateEdge), buffer.size(), files[bucket]) != buffer.size()) { + Debug(Debug::ERROR) << "Cannot write " << buffer.size() << " edges to bucket " << bucket + << " of " << dir << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + buffer.clear(); +} + +void EdgeBucketWriter::append(unsigned int bucket, const CandidateEdge &edge) { + buffers[bucket].push_back(edge); + if (buffers[bucket].size() >= edgesPerBuffer) { + flush(bucket); + } + edgeCount++; +} + +void EdgeBucketWriter::flushAll() { + for (unsigned int b = 0; b < bucketCount; b++) { + flush(b); + if (files[b] != NULL && fflush(files[b]) != 0) { + Debug(Debug::ERROR) << "Cannot flush edge bucket " << b << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + } +} + +void EdgeBucketWriter::close() { + if (closed) { + return; + } + closed = true; + for (unsigned int b = 0; b < bucketCount; b++) { + flush(b); + if (files[b] != NULL) { + if (fclose(files[b]) != 0) { + Debug(Debug::ERROR) << "Cannot close edge bucket " << b << "\n"; + EXIT(EXIT_FAILURE); + } + files[b] = NULL; + } + } +} + +std::vector EdgeBucketWriter::shardFiles(const std::string &dir, unsigned int bucket) { + const std::string path = bucketDir(dir, bucket); + std::vector shards; + DIR *handle = opendir(path.c_str()); + if (handle == NULL) { + return shards; // a bucket nothing was written to is empty, not an error + } + struct dirent *entry; + while ((entry = readdir(handle)) != NULL) { + const std::string name = entry->d_name; + if (name.size() > 6 && name.compare(name.size() - 6, 6, ".edges") == 0) { + shards.push_back(path + "/" + name); + } + } + closedir(handle); + std::sort(shards.begin(), shards.end()); + return shards; +} diff --git a/src/linclust/CandidateEdge.h b/src/linclust/CandidateEdge.h index fca6febc3..fe16af5fd 100644 --- a/src/linclust/CandidateEdge.h +++ b/src/linclust/CandidateEdge.h @@ -52,6 +52,9 @@ struct __attribute__((__packed__)) CandidateEdge { // One file per partition, written whole, so a partition redone after a crash // simply overwrites its file. That makes the reduce idempotent, unlike the map, // whose per-worker shards can retain a dead worker's partial output. +// +// Used for the reference layout (`--align`, small scale). Production uses +// EdgeBucketWriter below. class EdgeWriter { public: EdgeWriter(const std::string &path, size_t bufferRecords = 1024 * 1024); @@ -78,4 +81,57 @@ class EdgeWriter { bool closed; }; + +// Writes edges into buckets by *representative key range*. +// +// This is the layout the alignment stage needs, and the reason is not obvious: +// aligning inside the k-mer partition fails because a partition's pairs are +// scattered over the whole key space, so every partition ends up re-reading the +// entire sequence database (measured: 52x amplification, see DESIGN_DECISIONS.md +// §9). Bucketing by representative key gives each align worker a contiguous slice +// of sequences instead. +// +// Two things fall out for free: +// - every copy of a pair produced by different k-mer partitions lands in the +// same bucket, so the cross-partition duplicates are removed here rather than +// paid for in duplicate alignments; +// - having all copies together means the per-(pair, diagonal) score +// accumulation stock does in its global merge can be reproduced exactly, +// instead of approximated per partition. +// +// One file per (worker, bucket), like the k-mer shards, so no locking is needed. +class EdgeBucketWriter { +public: + EdgeBucketWriter(const std::string &dir, unsigned int bucketCount, const std::string &shardId, + size_t bufferBudgetBytes = 256 * 1024 * 1024); + ~EdgeBucketWriter(); + + void append(unsigned int bucket, const CandidateEdge &edge); + // Pushes buffered edges to the OS. Call before marking a work item done, for + // the same reason the k-mer writer does. + void flushAll(); + void close(); + + uint64_t getEdgeCount() const { return edgeCount; } + + static void createLayout(const std::string &dir, unsigned int bucketCount); + static std::string bucketDir(const std::string &dir, unsigned int bucket); + static std::vector shardFiles(const std::string &dir, unsigned int bucket); + +private: + EdgeBucketWriter(const EdgeBucketWriter &); + EdgeBucketWriter &operator=(const EdgeBucketWriter &); + + void flush(unsigned int bucket); + + std::string dir; + std::string shardId; + unsigned int bucketCount; + size_t edgesPerBuffer; + std::vector > buffers; + std::vector files; + uint64_t edgeCount; + bool closed; +}; + #endif diff --git a/src/linclust/PartitionSequences.cpp b/src/linclust/PartitionSequences.cpp new file mode 100644 index 000000000..0b4b46614 --- /dev/null +++ b/src/linclust/PartitionSequences.cpp @@ -0,0 +1,146 @@ +#include "PartitionSequences.h" + +#include "Debug.h" +#include "DenseIndex.h" +#include "Util.h" + +#include +#include +#include + +#include +#include + +namespace { +// Index entries within this distance are fetched in one read rather than two. +// The dense index is 12 B per entry, so this coalesces anything within ~5000 +// keys -- which, because keys are length-ranked, is a very short stretch of the +// length distribution and therefore extremely common between neighbouring hits. +const size_t INDEX_COALESCE_BYTES = 64 * 1024; +// Same idea on the data file, where entries are ~250 B, so this bridges gaps of +// a few thousand sequences. +const size_t DATA_COALESCE_BYTES = 1024 * 1024; +} // namespace + +PartitionSequences::PartitionSequences(const std::string &dbName) + : dbName(dbName), dataFd(-1), indexFd(-1), bytesRead(0) { + const DenseIndex::Info info = DenseIndex::readInfo(dbName); + entryCount = info.entryCount; + firstKey = info.firstKey; + + dataFd = open(dbName.c_str(), O_RDONLY); + if (dataFd < 0) { + Debug(Debug::ERROR) << "Cannot open " << dbName << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + const std::string indexName = DenseIndex::fileName(dbName); + indexFd = open(indexName.c_str(), O_RDONLY); + if (indexFd < 0) { + Debug(Debug::ERROR) << "Cannot open " << indexName << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } +} + +PartitionSequences::~PartitionSequences() { + if (dataFd >= 0) { + close(dataFd); + } + if (indexFd >= 0) { + close(indexFd); + } +} + +void PartitionSequences::readAt(int fd, void *dst, size_t length, size_t offset, const char *what) { + char *cursor = static_cast(dst); + size_t done = 0; + while (done < length) { + const ssize_t got = pread(fd, cursor + done, length - done, static_cast(offset + done)); + if (got < 0) { + if (errno == EINTR) { + continue; + } + Debug(Debug::ERROR) << "Cannot read " << what << " from " << dbName << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + if (got == 0) { + Debug(Debug::ERROR) << "Unexpected end of file reading " << what << " from " << dbName + << " at offset " << (offset + done) << "\n"; + EXIT(EXIT_FAILURE); + } + done += static_cast(got); + } + bytesRead += length; +} + +void PartitionSequences::load(const std::vector &sortedKeys) { + keys = sortedKeys; + offsets.assign(keys.size(), 0); + lengths.assign(keys.size(), 0); + arena.clear(); + bytesRead = 0; + if (keys.empty()) { + return; + } + + // Pass 1: the index entries, read in coalesced ascending runs. + std::vector entries(keys.size()); + size_t i = 0; + std::vector block; + while (i < keys.size()) { + const uint64_t firstRow = keys[i] - firstKey; + size_t j = i; + while (j + 1 < keys.size() && + (keys[j + 1] - keys[i]) * sizeof(DenseIndex::Entry) < INDEX_COALESCE_BYTES) { + j++; + } + const uint64_t lastRow = keys[j] - firstKey; + const size_t span = static_cast(lastRow - firstRow + 1) * sizeof(DenseIndex::Entry); + block.resize(span); + readAt(indexFd, block.data(), span, DenseIndex::entryOffset(firstRow), "index entries"); + for (size_t k = i; k <= j; k++) { + const size_t at = static_cast(keys[k] - firstKey - firstRow) * sizeof(DenseIndex::Entry); + memcpy(&entries[k], block.data() + at, sizeof(DenseIndex::Entry)); + } + i = j + 1; + } + + // Pass 2: the residues themselves, again in coalesced ascending runs. Dense + // keys make ascending key mean ascending offset, so this is a forward scan. + uint64_t total = 0; + for (size_t k = 0; k < keys.size(); k++) { + lengths[k] = entries[k].length > 0 ? entries[k].length - 1 : 0; // drop the trailing newline + offsets[k] = total; + total += lengths[k]; + } + arena.resize(total); + + i = 0; + while (i < keys.size()) { + size_t j = i; + while (j + 1 < keys.size() && + entries[j + 1].offset >= entries[j].offset && + (entries[j + 1].offset - entries[i].offset) < DATA_COALESCE_BYTES) { + j++; + } + const uint64_t from = entries[i].offset; + const uint64_t to = entries[j].offset + entries[j].length; + block.resize(static_cast(to - from)); + readAt(dataFd, block.data(), block.size(), from, "sequence data"); + for (size_t k = i; k <= j; k++) { + memcpy(arena.data() + offsets[k], block.data() + (entries[k].offset - from), lengths[k]); + } + i = j + 1; + } +} + +const char *PartitionSequences::get(uint64_t key, unsigned int *length) const { + const std::vector::const_iterator it = + std::lower_bound(keys.begin(), keys.end(), key); + if (it == keys.end() || *it != key) { + return NULL; + } + const size_t idx = static_cast(it - keys.begin()); + *length = lengths[idx]; + return arena.data() + offsets[idx]; +} diff --git a/src/linclust/PartitionSequences.h b/src/linclust/PartitionSequences.h new file mode 100644 index 000000000..8faa0d8e4 --- /dev/null +++ b/src/linclust/PartitionSequences.h @@ -0,0 +1,62 @@ +#ifndef MMSEQS_PARTITIONSEQUENCES_H +#define MMSEQS_PARTITIONSEQUENCES_H + +#include +#include +#include + +// The sequences one k-mer partition needs, fetched by key into a local arena. +// +// This is the piece that lets the alignment run inside the reduce without a +// resident index. Stock align2clust opens the whole sequence DB with USE_INDEX +// (Align2clust.cpp:400), which is 24 B per sequence -- 2.4 TB at 1e11 and 24 TB +// at 1e12, on a node that has 2 TB. Here only the keys a partition actually +// touches are read, addressed directly through the dense companion index. +// +// Reads are issued in ascending key order, which matters more than it looks: +// dense keys mean ascending key == ascending file offset, so a scattered set of +// sequences is fetched as a forward scan rather than as random seeks. Adjacent +// requests are coalesced into single reads for the same reason. +// +// Sizing is the open question this class exists to answer. A partition touches +// roughly `kmersPerSequence / P` of the database, so the arena is small at test +// scale and large at target scale; getBytes() is reported so the extrapolation +// is measured rather than guessed. +class PartitionSequences { +public: + explicit PartitionSequences(const std::string &dbName); + ~PartitionSequences(); + + // Fetches every key in `sortedKeys` (ascending, unique) into the arena, + // replacing whatever was there before. + void load(const std::vector &sortedKeys); + + // Residues for a key, or NULL if it was not loaded. The returned pointer is + // valid until the next load(). + const char *get(uint64_t key, unsigned int *length) const; + + uint64_t getBytes() const { return arena.size(); } + uint64_t getCount() const { return keys.size(); } + // Bytes actually read from disk, including coalesced gaps. + uint64_t getBytesRead() const { return bytesRead; } + +private: + PartitionSequences(const PartitionSequences &); + PartitionSequences &operator=(const PartitionSequences &); + + void readAt(int fd, void *dst, size_t length, size_t offset, const char *what); + + std::string dbName; + int dataFd; + int indexFd; + uint64_t entryCount; + uint64_t firstKey; + + std::vector keys; // ascending, parallel to offsets/lengths + std::vector offsets; // into arena + std::vector lengths; // residues, without the newline/terminator + std::vector arena; + uint64_t bytesRead; +}; + +#endif diff --git a/src/linclust/alignparallel.cpp b/src/linclust/alignparallel.cpp new file mode 100644 index 000000000..64b19e254 --- /dev/null +++ b/src/linclust/alignparallel.cpp @@ -0,0 +1,347 @@ +/* + * alignparallel -- the alignment stage of the distributed linclust. + * + * Claims one representative-key-range bucket of candidate edges at a time, + * merges the duplicate copies the k-mer partitions produced, aligns each + * surviving pair once, and writes the survivors. + * + * Why this is a separate stage rather than fused into the reduce, which is the + * obvious design and the one tried first: alignment needs both sequences of every + * pair, and a k-mer partition's pairs are scattered over the whole key space. + * Measured on UniRef100, a single partition needed 1.61 GB of sequences but read + * 83.9 GB to get them -- 52x amplification, effectively the whole database, once + * per partition. Bucketing by representative key instead gives each worker a + * contiguous slice of sequences, which is what makes the reads sequential. + * + * Two properties fall out of bucketing by representative, and both matter: + * + * - Every copy of a pair, from however many k-mer partitions found it, lands in + * the same bucket. So the cross-partition duplicates (measured factor 1.35) + * are removed *before* aligning rather than paid for as duplicate alignments. + * - Having all copies together means stock's global accumulation of score per + * (pair, diagonal) can be reproduced exactly -- pick the diagonal the most + * k-mers agree on across all partitions -- rather than approximated within one + * partition. + * + * Neither the k-mer buckets nor a resident sequence index is needed here; the + * sequences come through the dense companion index, addressed by key. + */ +#include "BlockAligner.h" +#include "CandidateEdge.h" +#include "Command.h" +#include "Debug.h" +#include "DenseIndex.h" +#include "EvalueComputation.h" +#include "FastSort.h" +#include "FileUtil.h" +#include "Matcher.h" +#include "ParallelCoordination.h" +#include "Parameters.h" +#include "PartitionSequences.h" +#include "Sequence.h" +#include "SubstitutionMatrix.h" +#include "Util.h" + +#include +#include +#include +#include +#include + +#ifdef OPENMP +#include +#endif + +namespace { + +bool compareEdgeByPairAndDiagonal(const CandidateEdge &a, const CandidateEdge &b) { + const uint64_t ra = a.getRep(), rb = b.getRep(); + if (ra != rb) return ra < rb; + const uint64_t ma = a.getMember(), mb = b.getMember(); + if (ma != mb) return ma < mb; + if (a.diagonal != b.diagonal) return a.diagonal < b.diagonal; + return a.reverseStrand < b.reverseStrand; +} + +// Collapses the copies of each pair into one edge, exactly as stock's merge does. +// +// Stock accumulates a score per (pair, diagonal) across every split and iteration +// and keeps the diagonal with the highest total (kmermatcher.cpp:1913-1925). Here +// the copies come from different k-mer partitions instead, but they are all +// present in this bucket, so the same accumulation is possible and the result is +// the same. Doing it per partition -- which is all the reduce can do -- gives a +// different diagonal for 0.86% of pairs; doing it here does not. +size_t mergePairCopies(std::vector &edges) { + if (edges.empty()) { + return 0; + } + SORT_PARALLEL(edges.begin(), edges.end(), compareEdgeByPairAndDiagonal); + + size_t out = 0; + size_t i = 0; + while (i < edges.size()) { + const uint64_t rep = edges[i].getRep(); + const uint64_t member = edges[i].getMember(); + + int bestTotal = -1; + int16_t bestDiagonal = edges[i].diagonal; + uint8_t bestStrand = edges[i].reverseStrand; + int runTotal = 0; + int16_t runDiagonal = edges[i].diagonal; + + size_t j = i; + while (j < edges.size() && edges[j].getRep() == rep && edges[j].getMember() == member) { + if (edges[j].diagonal != runDiagonal) { + runDiagonal = edges[j].diagonal; + runTotal = 0; + } + runTotal += edges[j].score; + // >= not >, so the largest diagonal wins a tie -- the same rule stock + // applies in its merge (kmermatcher.cpp:1913), reached because the + // edges are sorted by ascending diagonal. + if (runTotal >= bestTotal) { + bestTotal = runTotal; + bestDiagonal = runDiagonal; + bestStrand = edges[j].reverseStrand; + } + j++; + } + + edges[out] = edges[i]; + edges[out].diagonal = bestDiagonal; + edges[out].reverseStrand = bestStrand; + edges[out].score = static_cast(std::min(bestTotal, 255)); + out++; + i = j; + } + edges.resize(out); + return out; +} + +size_t readBucket(const std::string &edgeDir, unsigned int bucket, + std::vector &out) { + const std::vector shards = EdgeBucketWriter::shardFiles(edgeDir, bucket); + for (size_t i = 0; i < shards.size(); i++) { + const size_t bytes = FileUtil::getFileSize(shards[i]); + if (bytes % sizeof(CandidateEdge) != 0) { + Debug(Debug::ERROR) << "Edge shard " << shards[i] << " is " << bytes + << " bytes, not a whole number of edges. It was probably written " + << "by an interrupted worker.\n"; + EXIT(EXIT_FAILURE); + } + const size_t count = bytes / sizeof(CandidateEdge); + if (count == 0) { + continue; + } + FILE *file = FileUtil::openFileOrDie(shards[i].c_str(), "rb", true); + const size_t offset = out.size(); + out.resize(offset + count); + if (fread(out.data() + offset, sizeof(CandidateEdge), count, file) != count) { + Debug(Debug::ERROR) << "Cannot read edge shard " << shards[i] << "\n"; + EXIT(EXIT_FAILURE); + } + fclose(file); + } + return out.size(); +} + +} // namespace + +int alignparallel(int argc, const char **argv, const Command &command) { + Parameters &par = Parameters::getInstance(); + par.parseParameters(argc, argv, command, true, 0, MMseqsParameter::COMMAND_ALIGN); + + const std::string seqDb = par.db1; + const std::string edgeDir = par.db2; + const std::string alnDir = par.db3; + + const int dbType = FileUtil::parseDbType(seqDb.c_str()); + if (Parameters::isEqualDbtype(dbType, Parameters::DBTYPE_NUCLEOTIDES)) { + Debug(Debug::ERROR) << "alignparallel is implemented for amino acid databases only\n"; + EXIT(EXIT_FAILURE); + } + const DenseIndex::Info info = DenseIndex::readInfo(seqDb); + + const std::string manifestPath = edgeDir + "/coord/edge.info"; + if (FileUtil::fileExists(manifestPath.c_str()) == false) { + Debug(Debug::ERROR) << "No edge buckets at " << edgeDir << " (" << manifestPath + << " is missing). Run kmerreduceparallel first.\n"; + EXIT(EXIT_FAILURE); + } + unsigned int bucketCount = 0; + { + FILE *file = FileUtil::openFileOrDie(manifestPath.c_str(), "r", true); + char name[64]; + size_t value; + while (fscanf(file, "%63s\t%zu\n", name, &value) == 2) { + if (std::string(name) == "bucketCount") { + bucketCount = static_cast(value); + } + } + fclose(file); + } + if (bucketCount == 0) { + Debug(Debug::ERROR) << "Edge manifest " << manifestPath << " has no bucket count\n"; + EXIT(EXIT_FAILURE); + } + par.printParameters(command.cmd, argc, argv, *command.params); + + if (FileUtil::directoryExists(alnDir.c_str()) == false) { + FileUtil::makeDir(alnDir.c_str()); + } + const std::string coordDir = alnDir + "/coord"; + if (FileUtil::directoryExists(coordDir.c_str()) == false) { + FileUtil::makeDir(coordDir.c_str()); + } + + BaseMatrix *subMat = new SubstitutionMatrix(par.scoringMatrixFile.values.aminoacid().c_str(), 2.0, 0.0); + SubstitutionMatrix::FastMatrix fastMatrix = SubstitutionMatrix::createAsciiSubMat(*subMat); + const uint64_t residues = info.dataSize - 2 * info.entryCount; + EvalueComputation evaluer(residues, subMat); + const std::string library = (par.covMode == Parameters::COV_MODE_BIDIRECTIONAL) + ? getCovSeqidQscPercMinDiag() + : getCovSeqidQscPercMinDiagTargetCov(); + const float scorePerColThreshold = parsePrecisionLib(library, par.seqIdThr, par.covThr, 0.99); + const unsigned int maxSeqLen = info.maxSeqLen + 1; + Debug(Debug::INFO) << "Aligning " << bucketCount << " edge buckets; score-per-column cutoff " + << scorePerColThreshold << "\n"; + + SharedCounter workerCounter(coordDir + "/worker.counter"); + const int64_t workerId = workerCounter.fetchAdd(); + Debug(Debug::INFO) << "Worker " << workerId << " joined\n"; + + PartitionSequences sequences(seqDb); + uint64_t survivorCount = 0; + + { + WorkQueue queue(coordDir + "/align.queue", static_cast(bucketCount)); + const bool finished = queue.drain(workerId, [&](size_t bucket) { + std::vector edges; + readBucket(edgeDir, static_cast(bucket), edges); + const size_t raw = edges.size(); + if (raw == 0) { + EdgeWriter empty(EdgeWriter::partitionPath(alnDir, static_cast(bucket))); + empty.close(); + return; + } + + const size_t merged = mergePairCopies(edges); + + // Both endpoints, in ascending key order so the fetch is a forward + // scan. Representatives are contiguous within a bucket by + // construction; members are close to them because keys are + // length-ranked and homologues have similar lengths. + std::vector needed; + needed.reserve(edges.size() * 2); + for (size_t i = 0; i < edges.size(); i++) { + needed.push_back(edges[i].getRep()); + needed.push_back(edges[i].getMember()); + } + SORT_PARALLEL(needed.begin(), needed.end()); + needed.erase(std::unique(needed.begin(), needed.end()), needed.end()); + sequences.load(needed); + + // Edges are sorted by (rep, member), so a representative's edges are + // contiguous and its query profile is built once. + std::vector repStarts; + for (size_t i = 0; i < edges.size(); i++) { + if (i == 0 || edges[i].getRep() != edges[i - 1].getRep()) { + repStarts.push_back(i); + } + } + repStarts.push_back(edges.size()); + std::vector survives(edges.size(), 0); + +#pragma omp parallel num_threads(par.threads) + { + Sequence query(maxSeqLen, Parameters::DBTYPE_AMINO_ACIDS, subMat, 0, false, + par.compBiasCorrection); + Sequence target(maxSeqLen, Parameters::DBTYPE_AMINO_ACIDS, subMat, 0, false, + par.compBiasCorrection); + BlockAligner aligner(Parameters::DBTYPE_AMINO_ACIDS, maxSeqLen, subMat, &fastMatrix, + &evaluer, par.compBiasCorrection, par.compBiasCorrectionScale, + -par.gapOpen.values.aminoacid(), + -par.gapExtend.values.aminoacid()); + +#pragma omp for schedule(dynamic, 16) + for (size_t g = 0; g < repStarts.size() - 1; g++) { + const size_t from = repStarts[g]; + const size_t to = repStarts[g + 1]; + unsigned int repLen = 0; + const char *repSeq = sequences.get(edges[from].getRep(), &repLen); + if (repSeq == NULL || repLen == 0) { + continue; + } + query.mapSequence(0, static_cast(edges[from].getRep()), repSeq, repLen); + aligner.initQuery(&query); + + for (size_t e = from; e < to; e++) { + unsigned int memberLen = 0; + const char *memberSeq = sequences.get(edges[e].getMember(), &memberLen); + if (memberSeq == NULL || memberLen == 0) { + continue; + } + if (Util::canBeCovered(par.covThr, par.covMode, query.L, + static_cast(memberLen)) == false) { + continue; + } + target.mapSequence(0, static_cast(edges[e].getMember()), + memberSeq, memberLen); + + // Same predicates as stock align2clust (Align2clust.cpp:660-678). + BlockAligner::UngappedAln_res aln = + aligner.ungappedAlign(&target, edges[e].diagonal); + if (aln.eval > par.evalThr) continue; + if (aln.alnLen < par.alnLenThr) continue; + if (Util::hasCoverage(par.covThr, par.covMode, aln.qcov, aln.tcov) == false) continue; + + int identical = 0; + for (int q = aln.qStart; q <= aln.qEnd; q++) { + const char a = repSeq[q] & static_cast(~0x20); + const char b = memberSeq[aln.tStart + (q - aln.qStart)] & + static_cast(~0x20); + identical += (a == b) ? 1 : 0; + } + const float seqId = Util::computeSeqId(par.seqIdMode, identical, query.L, + target.L, aln.alnLen); + if (seqId < par.seqIdThr - std::numeric_limits::epsilon()) continue; + if (aln.diagonalLen > 0 && + static_cast(aln.score) / static_cast(aln.diagonalLen) < + scorePerColThreshold) { + continue; + } + // The greedy ranks by alignment score, not by k-mer count. + edges[e].score = static_cast(std::min(aln.bitScore, 255)); + survives[e] = 1; + } + } + } + + EdgeWriter writer(EdgeWriter::partitionPath(alnDir, static_cast(bucket))); + size_t kept = 0; + for (size_t i = 0; i < edges.size(); i++) { + if (survives[i]) { + writer.append(edges[i]); + kept++; + } + } + writer.close(); + __sync_fetch_and_add(&survivorCount, static_cast(kept)); + Debug(Debug::INFO) << "Bucket " << bucket << ": " << raw << " copies -> " << merged + << " pairs -> " << kept << " surviving, " << needed.size() + << " sequences (" << sequences.getBytes() / (1024 * 1024) + << " MB arena, " << sequences.getBytesRead() / (1024 * 1024) + << " MB read)\n"; + }); + if (finished == false) { + Debug(Debug::ERROR) << "Align stage stalled: work remains but no bucket is claimable\n"; + EXIT(EXIT_FAILURE); + } + } + + Debug(Debug::INFO) << "Worker " << workerId << " wrote " << survivorCount << " surviving edges\n"; + delete[] fastMatrix.matrix; + delete[] fastMatrix.matrixData; + delete subMat; + return EXIT_SUCCESS; +} diff --git a/src/linclust/kmerreduceparallel.cpp b/src/linclust/kmerreduceparallel.cpp index c62f5f017..7de4a2f0b 100644 --- a/src/linclust/kmerreduceparallel.cpp +++ b/src/linclust/kmerreduceparallel.cpp @@ -37,6 +37,10 @@ #include "Util.h" #include "kmermatcher.h" +#ifdef OPENMP +#include +#endif + #include #include #include @@ -159,14 +163,17 @@ void buildThreadOffsets(KmerPosition *positions, size_t count, in } -// Collapses one grouping round's output into (representative, member) edges. +// Collapses one grouping round's output into (representative, member, diagonal) +// edges, counting how many k-mers support each diagonal. // -// The same pair is produced once for every k-mer the two sequences share, so -// emitting them raw duplicates heavily -- measured 4.1 M records for 1.1 M -// distinct pairs on 1M sequences. Stock collapses them the same way in -// writeKmerMatcherResult (kmermatcher.cpp:1789): keep the diagonal that the most -// k-mers agree on, and use that count as the prefilter score. The array is sorted -// by (rep, member, diagonal), so both runs are contiguous. +// It deliberately does **not** pick a best diagonal here. Stock chooses the +// diagonal the most k-mers agree on *globally* (kmermatcher.cpp:1913-1925), and a +// partition only ever sees a fraction of a pair's k-mers. Collapsing to one +// diagonal per partition would discard the other diagonals' counts before the +// align stage can add them up -- measured, that reproduces stock's diagonal for +// only 98.64% of pairs. Emitting one edge per distinct diagonal keeps the counts +// intact so the global merge is exact, and costs ~12% more edges because a +// colinear pair puts all its k-mers on one diagonal anyway. template void collectRoundEdges(KmerPosition *grouped, size_t writePos, bool isNucleotide, std::vector &edges) { @@ -175,68 +182,59 @@ void collectRoundEdges(KmerPosition *grouped, size_t writePos, b while (i < writePos && grouped[i].kmer != SIZE_MAX) { const size_t repRaw = grouped[i].kmer; const DBKeyType member = grouped[i].id; - T bestDiagonal = grouped[i].pos; - size_t bestCount = 0; - T runDiagonal = grouped[i].pos; - size_t runCount = 0; size_t j = i; while (j < writePos && grouped[j].kmer == repRaw && grouped[j].id == member) { - if (grouped[j].pos == runDiagonal) { - runCount++; - } else { - runDiagonal = grouped[j].pos; - runCount = 1; + const T diagonal = grouped[j].pos; + size_t count = 0; + while (j < writePos && grouped[j].kmer == repRaw && grouped[j].id == member && + grouped[j].pos == diagonal) { + count++; + j++; + } + size_t rep = repRaw; + // Stock keeps the strand in bit 63 of the representative key; a 48-bit + // key has no room for it, so it moves into its own field. + edge.reverseStrand = 0; + if (isNucleotide) { + edge.reverseStrand = BIT_CHECK(rep, 63) == false; + rep = BIT_CLEAR(rep, 63); } - if (runCount > bestCount) { - bestCount = runCount; - bestDiagonal = runDiagonal; + // A sequence being its own representative carries no information: keys + // are dense, so any key never appearing as a member is its own + // representative by construction. + if (static_cast(rep) == member) { + continue; } - j++; + edge.setRep(static_cast(rep)); + edge.setMember(static_cast(member)); + edge.diagonal = static_cast(diagonal); + edge.score = static_cast(std::min(count, 255)); + edges.push_back(edge); } i = j; - - size_t rep = repRaw; - // Stock keeps the strand in bit 63 of the representative key; a 48-bit key - // has no room for it, so it moves into its own field. - edge.reverseStrand = 0; - if (isNucleotide) { - edge.reverseStrand = BIT_CHECK(rep, 63) == false; - rep = BIT_CLEAR(rep, 63); - } - // A sequence being its own representative carries no information here: - // keys are dense, so any key that never appears as a member is its own - // representative by construction, and the final greedy derives singletons - // from that rather than from a marker edge. - if (static_cast(rep) == member) { - continue; - } - edge.setRep(static_cast(rep)); - edge.setMember(static_cast(member)); - edge.diagonal = static_cast(bestDiagonal); - edge.score = static_cast(std::min(bestCount, 255)); - edges.push_back(edge); } } -// Orders edges so duplicates of a pair are adjacent, best score first. +// Orders edges so copies of the same (pair, diagonal) are adjacent. bool compareEdge(const CandidateEdge &a, const CandidateEdge &b) { const uint64_t ra = a.getRep(), rb = b.getRep(); if (ra != rb) return ra < rb; const uint64_t ma = a.getMember(), mb = b.getMember(); if (ma != mb) return ma < mb; - return a.score > b.score; + return a.diagonal < b.diagonal; } + + // Groups one partition and writes its edges. template -uint64_t reducePartition(const std::string &kmerDir, const std::string &edgeDir, - unsigned int partition, int dbType, Parameters &par, BaseMatrix *subMat) { +uint64_t reducePartition(const std::string &kmerDir, + unsigned int partition, int dbType, Parameters &par, BaseMatrix *subMat, + EdgeBucketWriter &writer, uint64_t bucketSpan) { const bool isNucleotide = Parameters::isEqualDbtype(dbType, Parameters::DBTYPE_NUCLEOTIDES); - EdgeWriter writer(EdgeWriter::partitionPath(edgeDir, partition)); const uint64_t recordCount = KmerBucketReader::countRecords(kmerDir, partition); if (recordCount == 0) { - writer.close(); return 0; } @@ -308,17 +306,31 @@ uint64_t reducePartition(const std::string &kmerDir, const std::string &edgeDir, // Rounds overlap heavily by construction, so collapse the union here rather // than writing the same pair once per round. Keeping the highest score means // the round that agreed on the most k-mers wins. + // The v2 rounds overlap heavily, so the same (pair, diagonal) is produced by + // several of them. Sum the supporting k-mer counts rather than keeping one, + // so the count the align stage accumulates globally stays meaningful. SORT_PARALLEL(edges.begin(), edges.end(), compareEdge); + size_t unique = 0; for (size_t i = 0; i < edges.size(); i++) { - if (i > 0 && edges[i].getRep() == edges[i - 1].getRep() && - edges[i].getMember() == edges[i - 1].getMember()) { + if (unique > 0 && edges[i].getRep() == edges[unique - 1].getRep() && + edges[i].getMember() == edges[unique - 1].getMember() && + edges[i].diagonal == edges[unique - 1].diagonal) { + edges[unique - 1].score = static_cast( + std::min(edges[unique - 1].score + edges[i].score, 255)); continue; } - writer.append(edges[i]); + edges[unique] = edges[i]; + unique++; } + edges.resize(unique); - writer.close(); - return writer.getEdgeCount(); + // Bucketed by representative key, which is what the align stage needs and + // what brings every copy of a pair together (see CandidateEdge.h). + for (size_t i = 0; i < edges.size(); i++) { + writer.append(static_cast(edges[i].getRep() / bucketSpan), edges[i]); + } + + return edges.size(); } } // namespace @@ -378,11 +390,42 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { FileUtil::makeDir(reduceCoordDir.c_str()); } + // Edge buckets are ranges of representative key. Sized so one bucket's + // sequences are a comfortable slice for the align stage, which loads them + // whole; the align workers' memory, not the reduce's, sets this. + const uint64_t targetBucketBytes = Util::computeMemory(par.splitMemoryLimit) / 4; + unsigned int bucketCount = 1; + while (bucketCount < 65536 && info.dataSize / bucketCount > targetBucketBytes) { + bucketCount *= 2; + } + const uint64_t bucketSpan = (info.entryCount + bucketCount - 1) / bucketCount; + + const std::string edgeManifest = reduceCoordDir + "/edge.info"; + { + FileLock lock(reduceCoordDir + "/edge.lock"); + lock.lock(); + if (FileUtil::fileExists(edgeManifest.c_str()) == false) { + EdgeBucketWriter::createLayout(edgeDir, bucketCount); + FILE *f = FileUtil::openAndDelete(edgeManifest.c_str(), "w"); + fprintf(f, "bucketCount\t%zu\n", (size_t)bucketCount); + fprintf(f, "bucketSpan\t%zu\n", (size_t)bucketSpan); + fprintf(f, "entryCount\t%zu\n", (size_t)info.entryCount); + fclose(f); + } + lock.unlock(); + } + Debug(Debug::INFO) << "Writing edges into " << bucketCount << " representative-key buckets of " + << bucketSpan << " keys\n"; + SharedCounter workerCounter(reduceCoordDir + "/worker.counter"); const int64_t workerId = workerCounter.fetchAdd(); Debug(Debug::INFO) << "Worker " << workerId << " joined\n"; BaseMatrix *subMat = createSubstitutionMatrix(par, dbType); + + EdgeBucketWriter *edgeWriter = + new EdgeBucketWriter(edgeDir, bucketCount, "w" + SSTR(workerId)); + uint64_t edgeCount = 0; { WorkQueue queue(reduceCoordDir + "/reduce.queue", static_cast(partitionCount)); @@ -390,14 +433,15 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { // partition are already threaded, and a partition is sized to fill a node. const bool finished = queue.drain(workerId, [&](size_t partition) { if (info.maxSeqLen < SHRT_MAX) { - edgeCount += reducePartition(kmerDir, edgeDir, - static_cast(partition), dbType, - par, subMat); + edgeCount += reducePartition(kmerDir, static_cast(partition), + dbType, par, subMat, *edgeWriter, bucketSpan); } else { - edgeCount += reducePartition(kmerDir, edgeDir, - static_cast(partition), dbType, par, - subMat); + edgeCount += reducePartition(kmerDir, static_cast(partition), dbType, + par, subMat, *edgeWriter, bucketSpan); } + // Before drain() records the bucket done, so a worker that dies never + // leaves an item complete whose edges were still buffered. + edgeWriter->flushAll(); }); if (finished == false) { Debug(Debug::ERROR) << "Reduce stage stalled: work remains but no partition is claimable\n"; @@ -405,7 +449,10 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { } } - Debug(Debug::INFO) << "Worker " << workerId << " wrote " << edgeCount << " candidate edges\n"; + Debug(Debug::INFO) << "Worker " << workerId << " wrote " << edgeCount + << " candidate edges\n"; + edgeWriter->close(); + delete edgeWriter; delete subMat; return EXIT_SUCCESS; From 21a07bebdb81974ddab6dd0a662b5e92394e04be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Thu, 30 Jul 2026 08:57:25 +0000 Subject: [PATCH 05/27] Add distributed align2clust. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Björn Buschkämper --- src/CommandDeclarations.h | 1 + src/MMseqsBase.cpp | 11 + src/commons/Parameters.cpp | 6 + src/commons/Parameters.h | 1 + src/linclust/CMakeLists.txt | 1 + src/linclust/PartitionSequences.cpp | 5 +- src/linclust/alignparallel.cpp | 101 +++++++-- src/linclust/greedycluster.cpp | 310 ++++++++++++++++++++++++++++ 8 files changed, 417 insertions(+), 19 deletions(-) create mode 100644 src/linclust/greedycluster.cpp diff --git a/src/CommandDeclarations.h b/src/CommandDeclarations.h index 455424e0d..a872297c4 100644 --- a/src/CommandDeclarations.h +++ b/src/CommandDeclarations.h @@ -30,6 +30,7 @@ extern int createdbparallel(int argc, const char **argv, const Command& command) extern int kmermatcherparallel(int argc, const char **argv, const Command& command); extern int kmerreduceparallel(int argc, const char **argv, const Command& command); extern int alignparallel(int argc, const char **argv, const Command& command); +extern int greedycluster(int argc, const char **argv, const Command& command); extern int makepaddedseqdb(int argc, const char **argv, const Command& command); extern int createindex(int argc, const char **argv, const Command& command); extern int createlinindex(int argc, const char **argv, const Command& command); diff --git a/src/MMseqsBase.cpp b/src/MMseqsBase.cpp index 905148de7..c4c7fe790 100644 --- a/src/MMseqsBase.cpp +++ b/src/MMseqsBase.cpp @@ -714,6 +714,17 @@ std::vector baseCommands = { CITATION_MMSEQS2, {{"sequenceDB", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::sequenceDb }, {"edgeDir", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::directory }, {"alnDir", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::directory }}}, + {"greedycluster", greedycluster, &par.greedycluster, COMMAND_CLUSTER, + "Greedy clustering over the surviving edges, in one key-ordered sweep", + "# Representatives always have lower keys than their members, so a single\n" + "# left-to-right sweep is the exact greedy. Needs two bits per key, not the\n" + "# eight bytes per sequence stock's fused clustering keeps resident.\n" + "mmseqs greedycluster sequenceDB alnDir clusters.tsv\n", + "Martin Steinegger ", + " ", + CITATION_MMSEQS2, {{"sequenceDB", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::sequenceDb }, + {"alnDir", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::directory }, + {"clusterTsv", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::flatfile }}}, {"kmersearch", kmersearch, &par.kmersearch, COMMAND_PREFILTER, "Find bottom-m-hashed k-mer matches between target and query DB", NULL, diff --git a/src/commons/Parameters.cpp b/src/commons/Parameters.cpp index 538c01edf..b8b40be07 100644 --- a/src/commons/Parameters.cpp +++ b/src/commons/Parameters.cpp @@ -980,6 +980,12 @@ Parameters::Parameters(): alignparallel.push_back(&PARAM_COMPRESSED); alignparallel.push_back(&PARAM_V); + // greedycluster + // No clustering-mode knob: the sweep implements linclust's greedy, which is + // the only mode the key ordering makes exact in one pass. + greedycluster.push_back(&PARAM_THREADS); + greedycluster.push_back(&PARAM_V); + // makepaddedseqdb makepaddedseqdb.push_back(&PARAM_SUB_MAT); makepaddedseqdb.push_back(&PARAM_SCORE_BIAS); diff --git a/src/commons/Parameters.h b/src/commons/Parameters.h index a40ab9014..40122524c 100644 --- a/src/commons/Parameters.h +++ b/src/commons/Parameters.h @@ -1240,6 +1240,7 @@ class Parameters { std::vector kmermatcherparallel; std::vector kmerreduceparallel; std::vector alignparallel; + std::vector greedycluster; std::vector makepaddedseqdb; std::vector convert2fasta; std::vector result2flat; diff --git a/src/linclust/CMakeLists.txt b/src/linclust/CMakeLists.txt index d45f24626..713a1338f 100644 --- a/src/linclust/CMakeLists.txt +++ b/src/linclust/CMakeLists.txt @@ -3,6 +3,7 @@ set(linclust_source_files linclust/kmermatcherparallel.cpp linclust/kmerreduceparallel.cpp linclust/alignparallel.cpp + linclust/greedycluster.cpp linclust/CandidateEdge.cpp linclust/PartitionSequences.cpp linclust/KmerPartition.cpp diff --git a/src/linclust/PartitionSequences.cpp b/src/linclust/PartitionSequences.cpp index 0b4b46614..3c79c18f8 100644 --- a/src/linclust/PartitionSequences.cpp +++ b/src/linclust/PartitionSequences.cpp @@ -109,7 +109,10 @@ void PartitionSequences::load(const std::vector &sortedKeys) { // keys make ascending key mean ascending offset, so this is a forward scan. uint64_t total = 0; for (size_t k = 0; k < keys.size(); k++) { - lengths[k] = entries[k].length > 0 ? entries[k].length - 1 : 0; // drop the trailing newline + // The dense index stores residues + 2: the '\n' and DBWriter's '\0' + // (createdbparallel.cpp:344). Both must come off, or the aligner sees the + // newline as a residue and every sequence is one too long. + lengths[k] = entries[k].length > 2 ? entries[k].length - 2 : 0; offsets[k] = total; total += lengths[k]; } diff --git a/src/linclust/alignparallel.cpp b/src/linclust/alignparallel.cpp index 64b19e254..1e4a1dd3a 100644 --- a/src/linclust/alignparallel.cpp +++ b/src/linclust/alignparallel.cpp @@ -26,6 +26,7 @@ * Neither the k-mer buckets nor a resident sequence index is needed here; the * sequences come through the dense companion index, addressed by key. */ +#include "Alignment.h" #include "BlockAligner.h" #include "CandidateEdge.h" #include "Command.h" @@ -203,6 +204,8 @@ int alignparallel(int argc, const char **argv, const Command &command) { : getCovSeqidQscPercMinDiagTargetCov(); const float scorePerColThreshold = parsePrecisionLib(library, par.seqIdThr, par.covThr, 0.99); const unsigned int maxSeqLen = info.maxSeqLen + 1; + // Same x-drop as stock (Align2clust.cpp:419, MIN_SIZE 32). + const int32_t xDrop = 32 * par.gapExtend.values.aminoacid() + par.gapOpen.values.aminoacid(); Debug(Debug::INFO) << "Aligning " << bucketCount << " edge buckets; score-per-column cutoff " << scorePerColThreshold << "\n"; @@ -288,31 +291,93 @@ int alignparallel(int argc, const char **argv, const Command &command) { target.mapSequence(0, static_cast(edges[e].getMember()), memberSeq, memberLen); - // Same predicates as stock align2clust (Align2clust.cpp:660-678). + // Stock's two-stage acceptance (Align2clust.cpp:660-792): an + // ungapped alignment on the k-mer diagonal, and if that + // fails, a gapped banded alignment seeded from a + // three-residue exact match. Leaving the second stage out + // silently drops every pair that needs gaps to align. BlockAligner::UngappedAln_res aln = aligner.ungappedAlign(&target, edges[e].diagonal); - if (aln.eval > par.evalThr) continue; - if (aln.alnLen < par.alnLenThr) continue; - if (Util::hasCoverage(par.covThr, par.covMode, aln.qcov, aln.tcov) == false) continue; - - int identical = 0; - for (int q = aln.qStart; q <= aln.qEnd; q++) { - const char a = repSeq[q] & static_cast(~0x20); - const char b = memberSeq[aln.tStart + (q - aln.qStart)] & - static_cast(~0x20); - identical += (a == b) ? 1 : 0; + + const bool hasEvalue = (aln.eval <= par.evalThr); + const bool hasAlnLen = (aln.alnLen >= par.alnLenThr); + const bool hasCoverage = + Util::hasCoverage(par.covThr, par.covMode, aln.qcov, aln.tcov); + float seqId = 0; + if (hasEvalue) { + int identical = 0; + for (int q = aln.qStart; q <= aln.qEnd; q++) { + const char a = repSeq[q] & static_cast(~0x20); + const char b = memberSeq[aln.tStart + (q - aln.qStart)] & + static_cast(~0x20); + identical += (a == b) ? 1 : 0; + } + seqId = Util::computeSeqId(par.seqIdMode, identical, query.L, target.L, + aln.alnLen); + } + const bool hasSeqId = + seqId >= (par.seqIdThr - std::numeric_limits::epsilon()); + + if (hasAlnLen && hasCoverage && hasSeqId && hasEvalue) { + // The greedy ranks by alignment score, not k-mer count. + edges[e].score = + static_cast(std::min(aln.bitScore, 255)); + survives[e] = 1; + continue; } - const float seqId = Util::computeSeqId(par.seqIdMode, identical, query.L, - target.L, aln.alnLen); - if (seqId < par.seqIdThr - std::numeric_limits::epsilon()) continue; - if (aln.diagonalLen > 0 && + + // Ungapped failed. score-per-column is the gate on paying + // for a gapped alignment -- it is *not* a filter on hits the + // ungapped stage already accepted. + if (aln.diagonalLen <= 0 || static_cast(aln.score) / static_cast(aln.diagonalLen) < scorePerColThreshold) { continue; } - // The greedy ranks by alignment score, not by k-mer count. - edges[e].score = static_cast(std::min(aln.bitScore, 255)); - survives[e] = 1; + if (aln.qStart == -1 || aln.tStart == -1 || aln.alnLen < 3) { + continue; + } + // Seed the band on the first three consecutive identities, + // starting one past it, exactly as stock does. + int seedQuery = static_cast(aln.qStart); + int seedTarget = static_cast(aln.tStart); + bool foundSeed = false; + for (int b = 0; b <= aln.alnLen - 3; b++) { + const int qp = static_cast(aln.qStart) + b; + const int tp = static_cast(aln.tStart) + b; + if (repSeq[qp] == memberSeq[tp] && repSeq[qp + 1] == memberSeq[tp + 1] && + repSeq[qp + 2] == memberSeq[tp + 2]) { + seedQuery = qp + 1; + seedTarget = tp + 1; + foundSeed = true; + break; + } + } + if (foundSeed == false) { + continue; + } + + std::string backtrace; + s_align gapped = aligner.bandedalign(&target, seedQuery, seedTarget, + backtrace, xDrop, par.covThr, + par.covMode); + const unsigned int gappedLen = backtrace.size(); + const double gappedSeqId = + Util::computeSeqId(par.seqIdMode, gapped.identicalAACnt, query.L, + memberLen, gappedLen); + Matcher::result_t result( + static_cast(edges[e].getMember()), gapped.score1, + gapped.qCov, gapped.tCov, gappedSeqId, gapped.evalue, gappedLen, + gapped.qStartPos1, gapped.qEndPos1, query.L, gapped.dbStartPos1, + gapped.dbEndPos1, memberLen, backtrace); + // isIdentity is always false here: self-edges are dropped in + // the reduce, so a representative is never its own member. + if (Alignment::checkCriteria(result, false, par.evalThr, par.seqIdThr, + par.alnLenThr, par.covMode, par.covThr)) { + edges[e].score = + static_cast(std::min(gapped.score1, 255)); + survives[e] = 1; + } } } } diff --git a/src/linclust/greedycluster.cpp b/src/linclust/greedycluster.cpp new file mode 100644 index 000000000..0c4c7f4d9 --- /dev/null +++ b/src/linclust/greedycluster.cpp @@ -0,0 +1,310 @@ +/* + * greedycluster -- the final clustering step of the distributed linclust. + * + * Reads the surviving edges, runs linclust's greedy, and writes the clustering. + * + * The whole stage rests on length-ranked keys. Stock's greedy walks sequences + * longest-first and makes each unassigned one a representative; key 0 is the + * longest sequence, so that order is exactly ascending key order. The greedy + * therefore collapses to a single left-to-right sweep -- no connected components, + * no iteration to a fixpoint, no cross-worker negotiation. + * + * The sweep must visit **every key**, not only the keys that appear as + * representatives in some edge. When it reaches key k unclaimed, k becomes a + * representative *there and then*, which is what stops a later, higher-keyed + * representative from claiming it. Skipping edge-less keys and sweeping them up + * as singletons afterwards is subtly wrong: with `--cov-mode 1` assignGroup emits + * reversed edges (kmermatcher.cpp, COV_MODE_TARGET branch), so a representative's + * key can exceed its member's, and those late claims would steal keys that stock + * had already made representatives. Measured, that mis-assigned 54 of 1,000,000 + * sequences. + * + * The second thing that falls out is the memory. Stock keeps + * `assignedCluster[dbSize]` at 8 bytes per sequence (Align2clust.cpp:445) -- + * 800 GB at 1e11 and 8 TB at 1e12, on a node that has 2 TB. Here the sweep needs + * only two bits per key: + * + * claimed -- some earlier representative took this sequence + * hasMembers -- this key is a representative that claimed at least one member + * + * which is 25 GB at 1e11 and 250 GB at 1e12, both resident on one node. The + * actual assignment never has to be stored, because the sweep visits + * representatives in increasing key order and can therefore emit each cluster + * exactly when it is finished. + * + * The sweep is sequential by necessity -- that is what makes it exact -- but the + * work per bucket (reading and sorting edges) is threaded, and the sweep itself + * is a bit test per edge. + */ +#include "CandidateEdge.h" +#include "Command.h" +#include "Debug.h" +#include "DenseIndex.h" +#include "FastSort.h" +#include "FileUtil.h" +#include "Parameters.h" +#include "Util.h" + +#include +#include +#include +#include +#include + +#include +#include + +#ifdef OPENMP +#include +#endif + +namespace { + +// One bit per key -- "decided": either claimed by a representative or made one. +// The only structure in this stage that scales with the database, at 12.5 GB for +// 1e11 sequences and 125 GB for 1e12, against stock's 8 bytes per sequence. +class KeyFlags { +public: + explicit KeyFlags(uint64_t keyCount) : words((keyCount + 63) / 64, 0) {} + + bool isDecided(uint64_t key) const { return (words[key >> 6] >> (key & 63)) & 1ULL; } + void decide(uint64_t key) { words[key >> 6] |= 1ULL << (key & 63); } + + uint64_t bytes() const { return words.size() * sizeof(uint64_t); } + +private: + std::vector words; +}; + +// Appends "a\tb\n". snprintf costs more than the rest of the sweep at scale. +inline void appendPair(std::string &out, uint64_t a, uint64_t b) { + char tmp[48]; + int n = 0; + char digits[24]; + int d = 0; + do { digits[d++] = static_cast('0' + (a % 10)); a /= 10; } while (a); + while (d) tmp[n++] = digits[--d]; + tmp[n++] = '\t'; + do { digits[d++] = static_cast('0' + (b % 10)); b /= 10; } while (b); + while (d) tmp[n++] = digits[--d]; + tmp[n++] = '\n'; + out.append(tmp, n); +} + +bool compareByRepThenMember(const CandidateEdge &a, const CandidateEdge &b) { + const uint64_t ra = a.getRep(), rb = b.getRep(); + if (ra != rb) return ra < rb; + return a.getMember() < b.getMember(); +} + +// Reads one bucket's edges with parallel pread. +// +// The sweep that consumes them has to be sequential, but this does not: at 1e11 +// sequences the surviving edges are ~7.6 TB and reading them is the dominant cost +// of the stage, so it is split across threads into a preallocated buffer. +size_t readBucket(const std::string &alnDir, unsigned int bucket, std::vector &out, + int threads) { + out.clear(); + const std::string path = EdgeWriter::partitionPath(alnDir, bucket); + if (FileUtil::fileExists(path.c_str()) == false) { + return 0; + } + const size_t bytes = FileUtil::getFileSize(path); + if (bytes % sizeof(CandidateEdge) != 0) { + Debug(Debug::ERROR) << "Edge file " << path << " is " << bytes + << " bytes, not a whole number of edges\n"; + EXIT(EXIT_FAILURE); + } + if (bytes == 0) { + return 0; + } + out.resize(bytes / sizeof(CandidateEdge)); + + const int fd = open(path.c_str(), O_RDONLY); + if (fd < 0) { + Debug(Debug::ERROR) << "Cannot open " << path << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + // Whole records per chunk, so no edge straddles two reads. + const size_t chunkBytes = 64 * 1024 * 1024; + const size_t chunks = (bytes + chunkBytes - 1) / chunkBytes; + char *base = reinterpret_cast(out.data()); + bool failed = false; +#pragma omp parallel for schedule(dynamic, 1) num_threads(threads) + for (size_t c = 0; c < chunks; c++) { + const size_t from = c * chunkBytes; + const size_t want = std::min(chunkBytes, bytes - from); + size_t done = 0; + while (done < want) { + const ssize_t got = pread(fd, base + from + done, want - done, + static_cast(from + done)); + if (got <= 0) { + if (got < 0 && errno == EINTR) { + continue; + } + failed = true; + break; + } + done += static_cast(got); + } + } + close(fd); + if (failed) { + Debug(Debug::ERROR) << "Cannot read " << path << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + return out.size(); +} + +} // namespace + +int greedycluster(int argc, const char **argv, const Command &command) { + Parameters &par = Parameters::getInstance(); + par.parseParameters(argc, argv, command, true, 0, MMseqsParameter::COMMAND_CLUST); + + const std::string seqDb = par.db1; + const std::string alnDir = par.db2; + const std::string outFile = par.db3; + + const DenseIndex::Info info = DenseIndex::readInfo(seqDb); + par.printParameters(command.cmd, argc, argv, *command.params); + + // Bucket count comes from the align stage's layout: one file per bucket, and + // buckets are ascending representative-key ranges, which is exactly the order + // the sweep needs. + unsigned int bucketCount = 0; + while (FileUtil::fileExists(EdgeWriter::partitionPath(alnDir, bucketCount).c_str())) { + bucketCount++; + } + if (bucketCount == 0) { + Debug(Debug::ERROR) << "No edge buckets in " << alnDir << ". Run alignparallel first.\n"; + EXIT(EXIT_FAILURE); + } + + KeyFlags flags(info.entryCount); + Debug(Debug::INFO) << "Sweeping " << info.entryCount << " keys over " << bucketCount + << " edge buckets, " << flags.bytes() / (1024 * 1024) << " MB of flags\n"; + + FILE *out = FileUtil::openAndDelete(outFile.c_str(), "w"); + std::string buffer; + buffer.reserve(64 * 1024 * 1024); + + uint64_t clusterCount = 0; + uint64_t assignedCount = 0; + std::vector edges; + + // Edge buckets are contiguous ascending representative-key ranges, the same + // ranges kmerreduceparallel derived, so walking buckets in order walks the key + // space in order. + const uint64_t bucketSpan = (info.entryCount + bucketCount - 1) / bucketCount; + + for (unsigned int bucket = 0; bucket < bucketCount; bucket++) { + const uint64_t lo = bucket * bucketSpan; + if (lo >= info.entryCount) { + break; + } + const uint64_t hi = std::min(lo + bucketSpan, info.entryCount); + readBucket(alnDir, bucket, edges, par.threads); + if (edges.empty() == false) { + SORT_PARALLEL(edges.begin(), edges.end(), compareByRepThenMember); + } + + size_t p = 0; + for (uint64_t key = lo; key < hi; key++) { + while (p < edges.size() && edges[p].getRep() < key) { + p++; + } + const size_t from = p; + while (p < edges.size() && edges[p].getRep() == key) { + p++; + } + if (flags.isDecided(key)) { + continue; + } + + // Stock forms a cluster only when at least two of its members are still + // unassigned -- the representative itself plus one other + // (Align2clust.cpp:362, `validMemberIds.size() <= 1`). A key whose + // partners have all been taken is therefore *not* made its own cluster + // here; it stays undecided and a later, higher-keyed representative may + // still claim it. Deciding it early costs 27 wrong assignments per + // million; never deciding it costs 54 the other way. + bool hasFreePartner = false; + for (size_t e = from; e < p; e++) { + if (flags.isDecided(edges[e].getMember()) == false) { + hasFreePartner = true; + break; + } + } + if (hasFreePartner == false) { + continue; + } + + flags.decide(key); + appendPair(buffer, key, key); + clusterCount++; + for (size_t e = from; e < p; e++) { + const uint64_t member = edges[e].getMember(); + if (flags.isDecided(member)) { + continue; + } + flags.decide(member); + appendPair(buffer, key, member); + assignedCount++; + } + if (buffer.size() > 32 * 1024 * 1024) { + fwrite(buffer.data(), 1, buffer.size(), out); + buffer.clear(); + } + } + } + if (buffer.empty() == false) { + fwrite(buffer.data(), 1, buffer.size(), out); + buffer.clear(); + } + + // Whatever the sweep never decided is a singleton. Each key is independent, so + // this is threaded; blocks keep the buffered text bounded, and `schedule(static)` + // hands out ascending contiguous ranges so thread order is key order. + uint64_t singletonCount = 0; + { + const uint64_t blockKeys = 64 * 1024 * 1024; + std::vector threadBuffers(par.threads); + for (uint64_t blockStart = 0; blockStart < info.entryCount; blockStart += blockKeys) { + const uint64_t blockEnd = std::min(blockStart + blockKeys, info.entryCount); + uint64_t blockSingletons = 0; +#pragma omp parallel num_threads(par.threads) reduction(+ : blockSingletons) + { + int tid = 0; +#ifdef OPENMP + tid = omp_get_thread_num(); +#endif + std::string &mine = threadBuffers[tid]; + mine.clear(); +#pragma omp for schedule(static) + for (uint64_t key = blockStart; key < blockEnd; key++) { + if (flags.isDecided(key) == false) { + appendPair(mine, key, key); + blockSingletons++; + } + } + } + for (int t = 0; t < par.threads; t++) { + if (threadBuffers[t].empty() == false) { + fwrite(threadBuffers[t].data(), 1, threadBuffers[t].size(), out); + } + } + singletonCount += blockSingletons; + } + } + + if (fclose(out) != 0) { + Debug(Debug::ERROR) << "Cannot close " << outFile << "\n"; + EXIT(EXIT_FAILURE); + } + + Debug(Debug::INFO) << clusterCount + singletonCount << " clusters (" << singletonCount + << " singletons), " << assignedCount + << " sequences assigned to another representative\n"; + return EXIT_SUCCESS; +} From fa62a16666a6ba2864c9e3e7ae9953402b0269ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Thu, 30 Jul 2026 12:01:25 +0000 Subject: [PATCH 06/27] Add k-mer extraction waves (to manage within scratch budget). MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Björn Buschkämper --- data/workflow/linclustparallel.sh | 178 +++++++++++++++++ src/CommandDeclarations.h | 3 + src/MMseqsBase.cpp | 33 ++++ src/commons/LengthRankedPlan.cpp | 42 ++++ src/commons/LengthRankedPlan.h | 18 +- src/commons/Parameters.cpp | 28 ++- src/commons/Parameters.h | 7 + src/linclust/CMakeLists.txt | 3 + src/linclust/KmerPartition.cpp | 21 +- src/linclust/KmerPartition.h | 13 +- src/linclust/alignparallel.cpp | 196 +++++++++++++++++- src/linclust/createrepdb.cpp | 248 +++++++++++++++++++++++ src/linclust/kmermatcherparallel.cpp | 38 +++- src/linclust/kmerreduceparallel.cpp | 62 +++++- src/linclust/mergeclusterparallel.cpp | 273 ++++++++++++++++++++++++++ src/linclust/translatecluster.cpp | 243 +++++++++++++++++++++++ src/test/TestKmerPartition.cpp | 20 ++ src/test/TestLengthRankedPlan.cpp | 70 +++++-- src/util/createdbparallel.cpp | 46 ++++- 19 files changed, 1507 insertions(+), 35 deletions(-) create mode 100755 data/workflow/linclustparallel.sh create mode 100644 src/linclust/createrepdb.cpp create mode 100644 src/linclust/mergeclusterparallel.cpp create mode 100644 src/linclust/translatecluster.cpp diff --git a/data/workflow/linclustparallel.sh b/data/workflow/linclustparallel.sh new file mode 100755 index 000000000..bc273dc29 --- /dev/null +++ b/data/workflow/linclustparallel.sh @@ -0,0 +1,178 @@ +#!/bin/sh -e +# Shared-filesystem parallel linclust. +# +# Runs the two linclust passes across many independent worker processes that +# coordinate only through files: no MPI, no rank argument, no node-to-node +# communication. Every worker of a stage runs a byte-identical command line, so a +# stage maps onto a Slurm array job and workers may join late, die, or be +# restarted. +# +# RUNNER="srun -n 64" ./linclustparallel.sh input.fasta clusters.tsv tmp +# +# Two things about this script are load-bearing and should not be "simplified": +# +# 1. **The parameters are derived once and passed consistently.** The stages do +# not re-derive them. `--cov-mode` must reach the *grouping* stage, because +# assignGroup uses it to decide edge orientation -- stock passes it to +# `kmermatcher`, which does extraction and grouping together, whereas here +# those are separate commands. `--kmer-per-seq 21` must be given explicitly +# because the standalone default derives 20, and `--min-seq-id` selects the +# k-mer length. Mismatching any of these silently produces a different, +# wrong clustering rather than an error. +# 2. **Pass 2 runs on a densely re-keyed representative database.** Every stage +# assumes dense keys, so `createrepdb` re-keys rather than sub-setting, and +# `translatecluster` maps the result back before the merge. +# +# Restart behaviour, which differs by stage and matters on a walltime-limited +# queue: +# - The multi-worker stages are **never skipped**. Their work queue is the +# source of truth, so re-running is how they resume; if everything is already +# done the workers see that and exit immediately. Guarding them on an output +# file would skip a half-finished stage. +# - The single-node stages are not resumable, so they are guarded on their +# output -- but they write to a temporary name and rename on success, so a +# half-written file is never mistaken for a finished one. Redoing one costs +# about an hour at 1e11, comfortably inside a 24 h walltime. +[ -z "$MMSEQS" ] && MMSEQS=mmseqs +[ -z "$RUNNER" ] && RUNNER="" +[ -z "$THREADS" ] && THREADS=$(nproc 2>/dev/null || echo 8) +[ -z "$MIN_SEQ_ID" ] && MIN_SEQ_ID=0.9 +[ -z "$COV" ] && COV=0.8 +[ -z "$COV_MODE" ] && COV_MODE=1 +[ -z "$EVAL" ] && EVAL=0.001 +[ -z "$SPLIT_MEMORY_LIMIT" ] && SPLIT_MEMORY_LIMIT=0 +[ -z "$SCRATCH_BUDGET" ] && SCRATCH_BUDGET=0 + +notExists() { [ ! -f "$1" ]; } +fail() { echo "Error: $1"; exit 1; } + +# Extraction waves. A wave re-extracts every k-mer but keeps only its own slice of +# partition space, so peak scratch is the whole shuffle divided by the wave count, +# paid for with that many passes over the sequences. How many are needed follows +# from --scratch-budget and is decided by the map, not here; the map writes it into +# the shuffle manifest, and wave 0 is always valid, so wave 0 runs first and the +# rest of the loop reads its count off the manifest. Each wave is reduced before +# the next is mapped, and the reduce drops the buckets it consumed -- running two +# waves' buckets at once is exactly what the budget said would not fit. +# +# The alignment does *not* run per wave. Waves partition k-mer space while edges +# are bucketed by representative, so every wave's surviving edges land in the same +# buckets and the align runs once, afterwards, over their union. +waveCount() { awk '$1 == "waveCount" { print $2 }' "$1/coord/shuffle.info"; } + +mapReduceWaves() { + # $1 sequence DB, $2 k-mer dir, $3 edge dir, $4... extra map arguments + _db="$1"; _kmer="$2"; _edges="$3"; shift 3 + # shellcheck disable=SC2086 + $RUNNER "$MMSEQS" kmermatcherparallel "$_db" "$_kmer" $KMER_COMMON "$@" --kmer-wave 0 \ + || fail "kmermatcherparallel died" + # shellcheck disable=SC2086 + $RUNNER "$MMSEQS" kmerreduceparallel "$_db" "$_kmer" "$_edges" $REDUCE_PAR --kmer-wave 0 \ + || fail "kmerreduceparallel died" + _waves=$(waveCount "$_kmer") + [ -z "$_waves" ] && fail "no wave count in $_kmer/coord/shuffle.info" + _w=1 + while [ "$_w" -lt "$_waves" ]; do + echo "--- extraction wave $_w of $_waves ---" + # shellcheck disable=SC2086 + $RUNNER "$MMSEQS" kmermatcherparallel "$_db" "$_kmer" $KMER_COMMON "$@" --kmer-wave $_w \ + || fail "kmermatcherparallel (wave $_w) died" + # shellcheck disable=SC2086 + $RUNNER "$MMSEQS" kmerreduceparallel "$_db" "$_kmer" "$_edges" $REDUCE_PAR --kmer-wave $_w \ + || fail "kmerreduceparallel (wave $_w) died" + _w=$((_w + 1)) + done +} + +[ "$#" -ne 3 ] && { echo "usage: [RUNNER=\"srun -n N\"] $0 "; exit 1; } +INPUT="$1" +OUT="$2" +TMP="$3" +mkdir -p "$TMP" + +# Shared by both passes. --min-seq-id belongs here because it selects the k-mer +# length and alphabet; coverage does not, because this stage only extracts k-mers. +KMER_COMMON="--alph-size aa:21,nucl:5 --min-seq-id $MIN_SEQ_ID --kmer-per-seq 21 \ + --mask 0 --mask-prob 0.9 --mask-lower-case 0 --mask-n-repeat 0 -k 0 --max-seq-len 65535 \ + --hash-shift 67 --ignore-multi-kmer 0 \ + --split-memory-limit $SPLIT_MEMORY_LIMIT --scratch-budget $SCRATCH_BUDGET --threads $THREADS" +REDUCE_PAR="-c $COV --cov-mode $COV_MODE --include-adjacency 1 --num-adjacency 3 \ + --split-memory-limit $SPLIT_MEMORY_LIMIT --threads $THREADS" +ALIGN_PAR="--min-seq-id $MIN_SEQ_ID --min-aln-len 0 --seq-id-mode 0 -e $EVAL -c $COV \ + --cov-mode $COV_MODE --threads $THREADS" + +# 0. Sequence database with dense, length-ranked keys. Every later stage depends +# on that ordering: it is what makes the greedy a single forward sweep. +DB="$INPUT" +if notExists "$INPUT.dbtype"; then + DB="$TMP/db" + if notExists "$DB.dbtype"; then + # shellcheck disable=SC2086 + $RUNNER "$MMSEQS" createdbparallel "$INPUT" "$DB" --threads $THREADS \ + || fail "createdbparallel died" + fi +fi + +# ---- pass 1, over the whole database -------------------------------------- +mapReduceWaves "$DB" "$TMP/kmer1" "$TMP/edges1" \ + --spaced-kmer-mode 0 --kmer-per-seq-scale aa:0.000,nucl:0.200 + +# shellcheck disable=SC2086 + $RUNNER "$MMSEQS" alignparallel "$DB" "$TMP/edges1" "$TMP/aln1" $ALIGN_PAR \ + || fail "alignparallel (pass 1) died" + +# Single node from here to the end of the pass: the greedy sweep is sequential by +# necessity, which is what makes it exact. +if notExists "$TMP/clu1.tsv"; then + # shellcheck disable=SC2086 + "$MMSEQS" greedycluster "$DB" "$TMP/aln1" "$TMP/clu1.tsv.tmp" --threads $THREADS \ + || fail "greedycluster (pass 1) died" + mv -f "$TMP/clu1.tsv.tmp" "$TMP/clu1.tsv" +fi + +# ---- pass 2, over the representatives -------------------------------------- +# Re-keyed densely rather than sub-set, because every stage addresses keys as +# array offsets. The key map translates the result back below. +if notExists "$TMP/rep.keymap"; then + # shellcheck disable=SC2086 + "$MMSEQS" createrepdb "$DB" "$TMP/clu1.tsv" "$TMP/rep" --threads $THREADS \ + || fail "createrepdb died" +fi + +mapReduceWaves "$TMP/rep" "$TMP/kmer2" "$TMP/edges2" \ + --spaced-kmer-mode 1 --kmer-per-seq-scale aa:0.100,nucl:0.100 + +# The filter gate: a representative may only take a member if every sequence of +# that member's pass-1 cluster also aligns to it. Needs the original database and +# the key map, since the clustering it consults is in original keys. +# shellcheck disable=SC2086 + $RUNNER "$MMSEQS" alignparallel "$TMP/rep" "$TMP/edges2" "$TMP/aln2" $ALIGN_PAR \ + --filter-cludb-file "$TMP/clu1.tsv" --filter-seqdb-file "$DB" \ + --key-map "$TMP/rep.keymap" \ + || fail "alignparallel (pass 2) died" + +if notExists "$TMP/clu2_sub.tsv"; then + # shellcheck disable=SC2086 + "$MMSEQS" greedycluster "$TMP/rep" "$TMP/aln2" "$TMP/clu2_sub.tsv.tmp" --threads $THREADS \ + || fail "greedycluster (pass 2) died" + mv -f "$TMP/clu2_sub.tsv.tmp" "$TMP/clu2_sub.tsv" +fi + +if notExists "$TMP/clu2.tsv"; then + # shellcheck disable=SC2086 + "$MMSEQS" translatecluster "$TMP/clu2_sub.tsv" "$TMP/rep.keymap" "$TMP/clu2.tsv.tmp" \ + --threads $THREADS --split-memory-limit $SPLIT_MEMORY_LIMIT \ + || fail "translatecluster died" + mv -f "$TMP/clu2.tsv.tmp" "$TMP/clu2.tsv" +fi + +# ---- fold pass 2 into pass 1 ------------------------------------------------ +if notExists "$OUT"; then + # shellcheck disable=SC2086 + "$MMSEQS" mergeclusterparallel "$DB" "$TMP/clu1.tsv" "$TMP/clu2.tsv" "$OUT.tmp" \ + --threads $THREADS --split-memory-limit $SPLIT_MEMORY_LIMIT \ + || fail "mergeclusterparallel died" + mv -f "$OUT.tmp" "$OUT" +fi + +echo "Wrote $OUT" diff --git a/src/CommandDeclarations.h b/src/CommandDeclarations.h index a872297c4..15e6660be 100644 --- a/src/CommandDeclarations.h +++ b/src/CommandDeclarations.h @@ -31,6 +31,9 @@ extern int kmermatcherparallel(int argc, const char **argv, const Command& comma extern int kmerreduceparallel(int argc, const char **argv, const Command& command); extern int alignparallel(int argc, const char **argv, const Command& command); extern int greedycluster(int argc, const char **argv, const Command& command); +extern int mergeclusterparallel(int argc, const char **argv, const Command& command); +extern int createrepdb(int argc, const char **argv, const Command& command); +extern int translatecluster(int argc, const char **argv, const Command& command); extern int makepaddedseqdb(int argc, const char **argv, const Command& command); extern int createindex(int argc, const char **argv, const Command& command); extern int createlinindex(int argc, const char **argv, const Command& command); diff --git a/src/MMseqsBase.cpp b/src/MMseqsBase.cpp index c4c7fe790..3ccd70896 100644 --- a/src/MMseqsBase.cpp +++ b/src/MMseqsBase.cpp @@ -725,6 +725,39 @@ std::vector baseCommands = { CITATION_MMSEQS2, {{"sequenceDB", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::sequenceDb }, {"alnDir", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::directory }, {"clusterTsv", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::flatfile }}}, + {"mergeclusterparallel", mergeclusterparallel, &par.mergeclusterparallel, COMMAND_CLUSTER, + "Compose the clusterings of successive linclust passes", + "# Folds a later clustering into an earlier one by a key-range join, so no\n" + "# per-sequence list array is needed (stock keeps 24 B/sequence of empty\n" + "# list headers before storing a single member).\n" + "mmseqs mergeclusterparallel sequenceDB pass1.tsv pass2.tsv clusters.tsv\n", + "Martin Steinegger ", + " ... ", + CITATION_MMSEQS2, {{"sequenceDB", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::sequenceDb }, + {"clusterTsv", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA|DbType::VARIADIC, &DbValidator::flatfile }, + {"clusterTsv", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::flatfile }}}, + {"createrepdb", createrepdb, &par.createrepdb, COMMAND_DATABASE_CREATION, + "Build a densely re-keyed representative DB for the next linclust pass", + "# Representatives keep length order, so sub-key i is the i-th representative\n" + "# and the copy is sequential. Writes .keymap (sub-key -> original key)\n" + "# so the next pass's clustering can be translated back before merging.\n" + "mmseqs createrepdb sequenceDB clusters.tsv repDB\n", + "Martin Steinegger ", + " ", + CITATION_MMSEQS2, {{"sequenceDB", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::sequenceDb }, + {"clusterTsv", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::flatfile }, + {"sequenceDB", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::flatfile }}}, + {"translatecluster", translatecluster, &par.translatecluster, COMMAND_CLUSTER, + "Rewrite a clustering from sub-key space into original keys", + "# The pass over a createrepdb sub-database returns sub-keys; this maps them\n" + "# back before merging. Each key column is translated in its own bucketed\n" + "# pass, so the key map is only ever read in contiguous slices.\n" + "mmseqs translatecluster pass2.tsv repDB.keymap pass2_orig.tsv\n", + "Martin Steinegger ", + " ", + CITATION_MMSEQS2, {{"clusterTsv", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::flatfile }, + {"keyMap", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::flatfile }, + {"clusterTsv", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::flatfile }}}, {"kmersearch", kmersearch, &par.kmersearch, COMMAND_PREFILTER, "Find bottom-m-hashed k-mer matches between target and query DB", NULL, diff --git a/src/commons/LengthRankedPlan.cpp b/src/commons/LengthRankedPlan.cpp index 069282ccb..a38fac7d8 100644 --- a/src/commons/LengthRankedPlan.cpp +++ b/src/commons/LengthRankedPlan.cpp @@ -45,6 +45,30 @@ void writeBlock(const std::string &path, const FileHeader &header, FileUtil::move(tmp.c_str(), path.c_str()); } +uint64_t decimalDigits(uint64_t value) { + uint64_t digits = 1; + while (value >= 10) { + value /= 10; + digits++; + } + return digits; +} + +// Total decimal digits of the numbers 0 .. n-1. A number x has one digit plus +// one more for every power of ten it reaches, so counting how many numbers below +// n reach each power gives the sum without iterating over the range -- which +// matters, since the range is the whole database. +uint64_t digitsBelow(uint64_t n) { + uint64_t total = n; + for (uint64_t power = 10; power <= n; power *= 10) { + total += n - power; + if (power > n / 10) { + break; // the next power would overflow + } + } + return total; +} + FileHeader readHeader(FILE *file, const std::string &path, uint64_t magic) { FileHeader header; if (fread(&header, sizeof(FileHeader), 1, file) != 1) { @@ -160,6 +184,13 @@ LengthRankedTotals buildLengthRankedPlan(std::vector &histograms uint64_t nextKey = 0; uint64_t nextDataOffset = 0; uint64_t nextHdrOffset = 0; + uint64_t nextLookupOffset = 0; + // Every chunk's sequences come from one input file, so the file index is the + // same on all of that chunk's lookup lines. + std::vector fileIdxDigits(histograms.size()); + for (size_t i = 0; i < histograms.size(); i++) { + fileIdxDigits[i] = decimalDigits(histograms[i].fileIdx); + } for (size_t l = 0; l < lengths.size(); l++) { const uint64_t length = lengths[l]; @@ -176,8 +207,18 @@ LengthRankedTotals buildLengthRankedPlan(std::vector &histograms entry.keyStart = nextKey; entry.dataOffset = nextDataOffset; entry.hdrOffset = nextHdrOffset; + entry.lookupOffset = nextLookupOffset; + // A `.lookup` line is "key\taccession\tfileIdx\n". The keys of this + // bucket are exactly nextKey .. nextKey + count - 1, so their total + // width is the difference of two prefix sums; everything else on the + // line is either fixed width or already counted in pass 1. + entry.lookupBytes = bucket.accessionBytes + + (digitsBelow(nextKey + bucket.count) - digitsBelow(nextKey)) + + bucket.count * (fileIdxDigits[i] + 3); plans[i].entries.push_back(entry); + nextLookupOffset += entry.lookupBytes; + nextKey += bucket.count; // A sequence of length L always occupies L + 2 data bytes, so this // stays exact without looking at the sequences themselves. @@ -202,6 +243,7 @@ LengthRankedTotals buildLengthRankedPlan(std::vector &histograms totals.seqCount = nextKey; totals.dataBytes = nextDataOffset; totals.headerBytes = nextHdrOffset; + totals.lookupBytes = nextLookupOffset; totals.maxSeqLen = lengths.empty() ? 0 : lengths[0]; return totals; } diff --git a/src/commons/LengthRankedPlan.h b/src/commons/LengthRankedPlan.h index 54264bc89..8781e7479 100644 --- a/src/commons/LengthRankedPlan.h +++ b/src/commons/LengthRankedPlan.h @@ -47,6 +47,13 @@ class ChunkHistogram { // that DBWriter appends. Summed here because header size cannot be // derived from sequence length the way data size can. uint64_t headerBytes; + // Total bytes of the accessions of those sequences, as + // Util::parseFastaHeader extracts them. Only this part of a `.lookup` + // line is unknowable without reading the input; the key and the file + // index are both decided by the plan, so the planner completes the line + // width itself. Recording finished line widths here instead would be + // circular -- the keys do not exist yet when pass 1 runs. + uint64_t accessionBytes; }; uint64_t chunkIdx; @@ -80,6 +87,13 @@ class ChunkPlan { uint64_t keyStart; uint64_t dataOffset; uint64_t hdrOffset; + // Where this bucket's `.lookup` lines go, and how many bytes they must + // occupy. The width is carried rather than re-derived so pass 2 can check + // the text it built against what the planner reserved: an offset scheme + // that is wrong by a byte would otherwise corrupt a neighbouring bucket + // silently. + uint64_t lookupOffset; + uint64_t lookupBytes; }; uint64_t chunkIdx; @@ -97,12 +111,14 @@ struct LengthRankedTotals { uint64_t seqCount; uint64_t dataBytes; uint64_t headerBytes; + uint64_t lookupBytes; uint64_t maxSeqLen; uint64_t nuclVotes; uint64_t sampleCount; LengthRankedTotals() - : seqCount(0), dataBytes(0), headerBytes(0), maxSeqLen(0), nuclVotes(0), sampleCount(0) {} + : seqCount(0), dataBytes(0), headerBytes(0), lookupBytes(0), maxSeqLen(0), nuclVotes(0), + sampleCount(0) {} }; // Turns the per-chunk histograms into per-chunk placement plans. diff --git a/src/commons/Parameters.cpp b/src/commons/Parameters.cpp index b8b40be07..a2e34dc39 100644 --- a/src/commons/Parameters.cpp +++ b/src/commons/Parameters.cpp @@ -56,6 +56,8 @@ Parameters::Parameters(): PARAM_SPLIT_MODE(PARAM_SPLIT_MODE_ID, "--split-mode", "Split mode", "0: split target db; 1: split query db; 2: auto, depending on main memory", typeid(int), (void *) &splitMode, "^[0-2]{1}$", MMseqsParameter::COMMAND_PREFILTER | MMseqsParameter::COMMAND_EXPERT), PARAM_SPLIT_MEMORY_LIMIT(PARAM_SPLIT_MEMORY_LIMIT_ID, "--split-memory-limit", "Split memory limit", "Set max memory per split. E.g. 800B, 5K, 10M, 1G. Default (0) to all available system memory", typeid(ByteParser), (void *) &splitMemoryLimit, "^(0|[1-9]{1}[0-9]*(B|K|M|G|T)?)$", MMseqsParameter::COMMAND_COMMON | MMseqsParameter::COMMAND_PREFILTER | MMseqsParameter::COMMAND_EXPERT), PARAM_CHUNK_SIZE(PARAM_CHUNK_SIZE_ID, "--chunk-size", "Chunk size", "Input bytes one work item covers. Smaller chunks spread work more evenly and use less memory per thread; larger chunks mean fewer coordination files. E.g. 64M, 256M, 1G", typeid(ByteParser), (void *) &chunkSize, "^([1-9]{1}[0-9]*(B|K|M|G|T)?)$", MMseqsParameter::COMMAND_EXPERT), + PARAM_KMER_WAVE(PARAM_KMER_WAVE_ID, "--kmer-wave", "K-mer wave", "Which extraction wave to write, when the scratch budget needs more than one. Each wave re-extracts every k-mer but keeps only its own slice of the partition space, so peak scratch is the whole shuffle divided by the wave count. Default -1 requires a single wave. Run waves 0..W-1, reducing each before starting the next.", typeid(int), (void *) &kmerWave, "^-?[0-9]+$", MMseqsParameter::COMMAND_EXPERT), + PARAM_KEY_MAP(PARAM_KEY_MAP_ID, "--key-map", "Sub-key map", "Maps this database's dense sub-keys back to the original keys, as written by createrepdb. Needed with --filter-cludb-file when the pass runs on a re-keyed representative database.", typeid(std::string), (void *) &keyMapFile, "", MMseqsParameter::COMMAND_ALIGN | MMseqsParameter::COMMAND_EXPERT), PARAM_SCRATCH_BUDGET(PARAM_SCRATCH_BUDGET_ID, "--scratch-budget", "Scratch budget", "Total scratch the run may occupy. The k-mer extraction wave count and the partition count are derived from this together with --split-memory-limit, rather than set by hand. Default (0) for a single wave. E.g. 100T, 500T", typeid(ByteParser), (void *) &scratchBudget, "^(0|[1-9]{1}[0-9]*(B|K|M|G|T)?)$", MMseqsParameter::COMMAND_EXPERT), PARAM_DISK_SPACE_LIMIT(PARAM_DISK_SPACE_LIMIT_ID, "--disk-space-limit", "Disk space limit", "Set max disk space to use for reverse profile searches. E.g. 800B, 5K, 10M, 1G. Default (0) to all available disk space in the temp folder", typeid(ByteParser), (void *) &diskSpaceLimit, "^(0|[1-9]{1}[0-9]*(B|K|M|G|T)?)$", MMseqsParameter::COMMAND_COMMON | MMseqsParameter::COMMAND_PREFILTER | MMseqsParameter::COMMAND_EXPERT), PARAM_SPLIT_AMINOACID(PARAM_SPLIT_AMINOACID_ID, "--split-aa", "Split by amino acid", "Try to find the best split boundaries by entry lengths", typeid(bool), (void *) &splitAA, "$", MMseqsParameter::COMMAND_EXPERT), @@ -912,9 +914,7 @@ Parameters::Parameters(): createdb.push_back(&PARAM_V); // createdbparallel - // No PARAM_WRITE_LOOKUP: .lookup is not emitted yet. It is variable-width and - // in key order, so it cannot be written at computed offsets like everything - // else; see PARALLEL_LINCLUST_WIP.md for the reconstruction sketch. + createdbparallel.push_back(&PARAM_WRITE_LOOKUP); createdbparallel.push_back(&PARAM_DB_TYPE); createdbparallel.push_back(&PARAM_CHUNK_SIZE); createdbparallel.push_back(&PARAM_THREADS); @@ -943,6 +943,7 @@ Parameters::Parameters(): kmermatcherparallel.push_back(&PARAM_IGNORE_MULTI_KMER); kmermatcherparallel.push_back(&PARAM_SPLIT_MEMORY_LIMIT); kmermatcherparallel.push_back(&PARAM_SCRATCH_BUDGET); + kmermatcherparallel.push_back(&PARAM_KMER_WAVE); kmermatcherparallel.push_back(&PARAM_THREADS); kmermatcherparallel.push_back(&PARAM_COMPRESSED); kmermatcherparallel.push_back(&PARAM_V); @@ -959,6 +960,7 @@ Parameters::Parameters(): kmerreduceparallel.push_back(&PARAM_SPLIT_MEMORY_LIMIT); kmerreduceparallel.push_back(&PARAM_INCLUDE_ADJACENCY); kmerreduceparallel.push_back(&PARAM_NUM_ADJACENCY); + kmerreduceparallel.push_back(&PARAM_KMER_WAVE); kmerreduceparallel.push_back(&PARAM_THREADS); kmerreduceparallel.push_back(&PARAM_COMPRESSED); kmerreduceparallel.push_back(&PARAM_V); @@ -976,6 +978,9 @@ Parameters::Parameters(): alignparallel.push_back(&PARAM_GAP_OPEN); alignparallel.push_back(&PARAM_GAP_EXTEND); alignparallel.push_back(&PARAM_NO_COMP_BIAS_CORR); + alignparallel.push_back(&PARAM_FILTER_CLUDB_FILE); + alignparallel.push_back(&PARAM_FILTER_SEQDB_FILE); + alignparallel.push_back(&PARAM_KEY_MAP); alignparallel.push_back(&PARAM_THREADS); alignparallel.push_back(&PARAM_COMPRESSED); alignparallel.push_back(&PARAM_V); @@ -986,6 +991,21 @@ Parameters::Parameters(): greedycluster.push_back(&PARAM_THREADS); greedycluster.push_back(&PARAM_V); + // mergeclusterparallel + mergeclusterparallel.push_back(&PARAM_SPLIT_MEMORY_LIMIT); + mergeclusterparallel.push_back(&PARAM_THREADS); + mergeclusterparallel.push_back(&PARAM_V); + + // createrepdb + createrepdb.push_back(&PARAM_THREADS); + createrepdb.push_back(&PARAM_COMPRESSED); + createrepdb.push_back(&PARAM_V); + + // translatecluster + translatecluster.push_back(&PARAM_SPLIT_MEMORY_LIMIT); + translatecluster.push_back(&PARAM_THREADS); + translatecluster.push_back(&PARAM_V); + // makepaddedseqdb makepaddedseqdb.push_back(&PARAM_SUB_MAT); makepaddedseqdb.push_back(&PARAM_SCORE_BIAS); @@ -2577,6 +2597,8 @@ void Parameters::setDefaults() { splitMode = DETECT_BEST_DB_SPLIT; splitMemoryLimit = 0; chunkSize = 256 * 1024 * 1024; + kmerWave = -1; + keyMapFile = ""; scratchBudget = 0; diskSpaceLimit = 0; splitAA = false; diff --git a/src/commons/Parameters.h b/src/commons/Parameters.h index 40122524c..d7f57fa75 100644 --- a/src/commons/Parameters.h +++ b/src/commons/Parameters.h @@ -440,6 +440,8 @@ class Parameters { int splitMode; // Split by query or target DB size_t splitMemoryLimit; // Maximum memory in bytes a split can use size_t chunkSize; // Input bytes one createdbparallel work item covers + std::string keyMapFile; // alignparallel: sub-key -> original key for the filter gate + int kmerWave; // kmermatcherparallel: which extraction wave to write size_t scratchBudget; // Scratch ceiling the k-mer wave count is derived from size_t diskSpaceLimit; // Maximum disk space in bytes for sliced reverse profile search bool splitAA; // Split database by amino acid count instead @@ -830,6 +832,8 @@ class Parameters { PARAMETER(PARAM_SPLIT_MODE) PARAMETER(PARAM_SPLIT_MEMORY_LIMIT) PARAMETER(PARAM_CHUNK_SIZE) + PARAMETER(PARAM_KMER_WAVE) + PARAMETER(PARAM_KEY_MAP) PARAMETER(PARAM_SCRATCH_BUDGET) PARAMETER(PARAM_DISK_SPACE_LIMIT) PARAMETER(PARAM_SPLIT_AMINOACID) @@ -1241,6 +1245,9 @@ class Parameters { std::vector kmerreduceparallel; std::vector alignparallel; std::vector greedycluster; + std::vector mergeclusterparallel; + std::vector createrepdb; + std::vector translatecluster; std::vector makepaddedseqdb; std::vector convert2fasta; std::vector result2flat; diff --git a/src/linclust/CMakeLists.txt b/src/linclust/CMakeLists.txt index 713a1338f..941f3131a 100644 --- a/src/linclust/CMakeLists.txt +++ b/src/linclust/CMakeLists.txt @@ -4,6 +4,9 @@ set(linclust_source_files linclust/kmerreduceparallel.cpp linclust/alignparallel.cpp linclust/greedycluster.cpp + linclust/mergeclusterparallel.cpp + linclust/createrepdb.cpp + linclust/translatecluster.cpp linclust/CandidateEdge.cpp linclust/PartitionSequences.cpp linclust/KmerPartition.cpp diff --git a/src/linclust/KmerPartition.cpp b/src/linclust/KmerPartition.cpp index 92a90c5d1..7dd3e8883 100644 --- a/src/linclust/KmerPartition.cpp +++ b/src/linclust/KmerPartition.cpp @@ -60,7 +60,14 @@ KmerShuffleSizing deriveKmerShuffleSizing(uint64_t sequenceCount, unsigned int k EXIT(EXIT_FAILURE); } const uint64_t available = scratchBudgetBytes - persistentBytes; - sizing.waveCount = static_cast( + // Rounded up to a power of two so the wave count divides the partition + // count exactly. A wave owns a contiguous slice of partitions, so if the + // count did not divide P the largest slice would exceed totalKmerBytes / + // waveCount and peak scratch would quietly exceed the budget -- and with + // P below the wave count, the last waves would own nothing at all while + // the first still held everything. At most this doubles the number of + // extraction passes; overrunning the scratch filesystem kills the run. + sizing.waveCount = roundUpToPowerOfTwo( std::max(divideRoundingUp(sizing.totalKmerBytes, available), 1)); } sizing.bytesPerWave = divideRoundingUp(sizing.totalKmerBytes, sizing.waveCount); @@ -71,6 +78,8 @@ KmerShuffleSizing deriveKmerShuffleSizing(uint64_t sequenceCount, unsigned int k sizing.partitionCount = roundUpToPowerOfTwo(divideRoundingUp(sizing.bytesPerWave, workerMemoryBytes)); } + // Both are powers of two, so this makes the wave count a divisor of P. + sizing.partitionCount = std::max(sizing.partitionCount, sizing.waveCount); if (sizing.partitionCount > 65536) { // Above the 16-bit hash space the partitioner cannot tell partitions // apart, so this is a real dead end rather than something to clamp: the @@ -109,9 +118,11 @@ void KmerBucketWriter::createLayout(const std::string &dir, unsigned int partiti } KmerBucketWriter::KmerBucketWriter(const std::string &dir, unsigned int partitionCount, - const std::string &shardId, size_t bufferBudgetBytes) + const std::string &shardId, size_t bufferBudgetBytes, + unsigned int partitionFrom, unsigned int partitionTo) : dir(dir), shardId(shardId), partitionCount(partitionCount), - mutexes(partitionCount) { + mutexes(partitionCount), partitionFrom(partitionFrom), + partitionTo(partitionTo == 0 ? partitionCount : partitionTo) { // At least a handful of records per partition even with a tiny budget, so a // large partition count degrades to more frequent flushes rather than to // one write syscall per k-mer. @@ -152,6 +163,10 @@ void KmerBucketWriter::flush(unsigned int partition) { } void KmerBucketWriter::append(unsigned int partition, const KmerRecord &record) { + // Outside this wave's slice: another wave re-extracts and writes it. + if (partition < partitionFrom || partition >= partitionTo) { + return; + } std::lock_guard guard(mutexes[partition]); buffers[partition].push_back(record); if (buffers[partition].size() >= recordsPerBuffer) { diff --git a/src/linclust/KmerPartition.h b/src/linclust/KmerPartition.h index 690d51324..8d7e72104 100644 --- a/src/linclust/KmerPartition.h +++ b/src/linclust/KmerPartition.h @@ -100,6 +100,9 @@ struct __attribute__((__packed__)) KmerRecord { // The two numbers that shape the k-mer shuffle, and where they came from. struct KmerShuffleSizing { uint64_t totalKmerBytes; // every k-mer record, summed over all waves + // Both powers of two, and waveCount divides partitionCount: a wave owns the + // contiguous slice [w * P / W, (w + 1) * P / W) of partition space, and the + // slices must be equal for peak scratch to actually be 1 / W of the whole. unsigned int waveCount; // extraction passes, so peak scratch stays in budget unsigned int partitionCount; // P uint64_t bytesPerWave; // peak k-mer bytes on disk at any one time @@ -193,8 +196,14 @@ class KmerBucketWriter { // bufferBudgetBytes is split evenly across partitions, so memory is bounded // regardless of partition count. Records accumulate per partition and are // flushed in large contiguous appends rather than one write per k-mer. + // partitionFrom/partitionTo restrict writing to one wave's slice of the + // partition space. A wave re-extracts every k-mer but keeps only its own + // slice, so peak scratch is the whole shuffle divided by the wave count -- + // which is what makes 1e12 fit a budget that cannot hold 504 TB at once. + // Appends outside the window are dropped; the wave that owns them writes them. KmerBucketWriter(const std::string &dir, unsigned int partitionCount, - const std::string &shardId, size_t bufferBudgetBytes = 1024 * 1024 * 1024); + const std::string &shardId, size_t bufferBudgetBytes = 1024 * 1024 * 1024, + unsigned int partitionFrom = 0, unsigned int partitionTo = 0); ~KmerBucketWriter(); // Thread-safe. @@ -235,6 +244,8 @@ class KmerBucketWriter { std::vector files; std::vector mutexes; std::vector recordCounts; + unsigned int partitionFrom; + unsigned int partitionTo; // exclusive }; // Reads every shard of one partition back. diff --git a/src/linclust/alignparallel.cpp b/src/linclust/alignparallel.cpp index 1e4a1dd3a..809e95452 100644 --- a/src/linclust/alignparallel.cpp +++ b/src/linclust/alignparallel.cpp @@ -27,6 +27,7 @@ * sequences come through the dense companion index, addressed by key. */ #include "Alignment.h" +#include "Matcher.h" #include "BlockAligner.h" #include "CandidateEdge.h" #include "Command.h" @@ -146,6 +147,138 @@ size_t readBucket(const std::string &edgeDir, unsigned int bucket, return out.size(); } + +// The pass-2 acceptance gate stock applies with --filter-cludb-file. +// +// Before a representative q may take a member t, stock additionally requires that +// **every sequence of t's pass-1 cluster** also aligns to q (Align2clust.cpp:657-730, +// the `allpass` loop). It is strictly more conservative than the plain alignment, +// so leaving it out merges too much: measured, 902,641 clusters against stock's +// 902,795. +// +// t is a pass-1 representative, so it is dense in *sub-key* space -- which makes the +// lookup a CSR index over sub-keys rather than anything indexed by the full key +// space: an offset per representative plus one key per sequence. +struct FilterGate { + std::vector keymap; // sub-key -> original key + std::vector clusterStart; // sub-key -> offset into members + std::vector members; // original keys, grouped by pass-1 cluster + PartitionSequences fullSeqs; + + FilterGate(const std::string &fullDb) : fullSeqs(fullDb) {} + + size_t size(uint64_t sub) const { return clusterStart[sub + 1] - clusterStart[sub]; } + + void load(const std::string &keymapFile, const std::string &pass1Tsv) { + const size_t bytes = FileUtil::getFileSize(keymapFile); + keymap.resize(bytes / sizeof(uint64_t)); + FILE *m = FileUtil::openFileOrDie(keymapFile.c_str(), "rb", true); + if (fread(keymap.data(), sizeof(uint64_t), keymap.size(), m) != keymap.size()) { + Debug(Debug::ERROR) << "Cannot read " << keymapFile << "\n"; + EXIT(EXIT_FAILURE); + } + fclose(m); + + // Two streaming passes: count, then fill. The representative of a pass-1 + // cluster is looked up in the ascending key map by binary search. + std::vector counts(keymap.size() + 1, 0); + for (int pass = 0; pass < 2; pass++) { + FILE *f = FileUtil::openFileOrDie(pass1Tsv.c_str(), "r", true); + char *line = NULL; + size_t cap = 0; + while (getline(&line, &cap, f) > 0) { + char *tab = strchr(line, '\t'); + if (tab == NULL) continue; + const uint64_t rep = strtoull(line, NULL, 10); + const uint64_t member = strtoull(tab + 1, NULL, 10); + const std::vector::const_iterator it = + std::lower_bound(keymap.begin(), keymap.end(), rep); + if (it == keymap.end() || *it != rep) continue; + const size_t sub = static_cast(it - keymap.begin()); + if (pass == 0) { + counts[sub]++; + } else { + members[clusterStart[sub] + counts[sub]] = member; + counts[sub]++; + } + } + free(line); + fclose(f); + if (pass == 0) { + clusterStart.assign(keymap.size() + 1, 0); + uint64_t total = 0; + for (size_t i = 0; i < keymap.size(); i++) { + clusterStart[i] = total; + total += counts[i]; + } + clusterStart[keymap.size()] = total; + members.assign(total, 0); + counts.assign(keymap.size() + 1, 0); + } + } + } +}; + + +// Runs stock's allpass loop for one candidate member. +// +// The member's whole pass-1 cluster must align to the representative, each element +// on diagonal 0 -- ungapped first, then a full Smith-Waterman if that fails, exactly +// as Align2clust.cpp:669-726. A singleton pass-1 cluster passes trivially. +bool passesFilterGate(const FilterGate *gate, uint64_t subMember, Sequence &query, + Sequence &element, BlockAligner &aligner, Matcher &matcher, + Parameters &par, unsigned int swMode) { + if (gate == NULL || subMember >= gate->keymap.size()) { + return true; + } + if (gate->size(subMember) <= 1) { + return true; // stock only runs the loop when numClu > 1 + } + const uint64_t targetKey = gate->keymap[subMember]; + for (uint64_t j = gate->clusterStart[subMember]; j < gate->clusterStart[subMember + 1]; j++) { + const uint64_t elementKey = gate->members[j]; + if (elementKey == targetKey) { + continue; + } + unsigned int elementLen = 0; + const char *elementSeq = gate->fullSeqs.get(elementKey, &elementLen); + if (elementSeq == NULL || elementLen == 0) { + return false; + } + element.mapSequence(0, static_cast(elementKey), elementSeq, elementLen); + if (Util::canBeCovered(par.covThr, par.covMode, query.L, element.L) == false) { + return false; + } + const short elementDiagonal = 0; + BlockAligner::UngappedAln_res ua = aligner.ungappedAlign(&element, elementDiagonal); + const bool hasEvalue = (ua.eval <= par.evalThr); + const bool hasAlnLen = (ua.alnLen >= par.alnLenThr); + const bool hasCoverage = Util::hasCoverage(par.covThr, par.covMode, ua.qcov, ua.tcov); + int identical = 0; + for (int q = ua.qStart; q <= ua.qEnd; q++) { + const char a = query.getSeqData()[q] & static_cast(~0x20); + const char b = elementSeq[ua.tStart + (q - ua.qStart)] & static_cast(~0x20); + identical += (a == b) ? 1 : 0; + } + const float elementSeqId = + Util::computeSeqId(par.seqIdMode, identical, query.L, elementLen, ua.alnLen); + const bool hasSeqId = + elementSeqId >= (par.seqIdThr - std::numeric_limits::epsilon()); + + if (hasAlnLen && hasCoverage && hasSeqId && hasEvalue) { + continue; + } + Matcher::result_t res = matcher.getSWResult(&element, static_cast(elementDiagonal), + false, par.covMode, par.covThr, par.evalThr, + swMode, par.seqIdMode, false); + if (Alignment::checkCriteria(res, false, par.evalThr, par.seqIdThr, par.alnLenThr, + par.covMode, par.covThr) == false) { + return false; + } + } + return true; +} + } // namespace int alignparallel(int argc, const char **argv, const Command &command) { @@ -158,7 +291,14 @@ int alignparallel(int argc, const char **argv, const Command &command) { const int dbType = FileUtil::parseDbType(seqDb.c_str()); if (Parameters::isEqualDbtype(dbType, Parameters::DBTYPE_NUCLEOTIDES)) { - Debug(Debug::ERROR) << "alignparallel is implemented for amino acid databases only\n"; + // Not a gap in the port: stock's align2clust, which this stage is the + // parallel form of, builds a protein matrix and protein Sequences + // unconditionally (Align2clust.cpp:406-520). Nucleotide linclust runs the + // v1 path (rescorediagonal) instead, which has no v2 counterpart to port. + // Erroring is therefore stricter than stock, which would silently score + // DNA with BLOSUM. + Debug(Debug::ERROR) << "alignparallel is amino acid only, as stock align2clust is. " + << "Nucleotide linclust uses the v1 alignment path.\n"; EXIT(EXIT_FAILURE); } const DenseIndex::Info info = DenseIndex::readInfo(seqDb); @@ -203,9 +343,30 @@ int alignparallel(int argc, const char **argv, const Command &command) { ? getCovSeqidQscPercMinDiag() : getCovSeqidQscPercMinDiagTargetCov(); const float scorePerColThreshold = parsePrecisionLib(library, par.seqIdThr, par.covThr, 0.99); - const unsigned int maxSeqLen = info.maxSeqLen + 1; + unsigned int maxSeqLen = info.maxSeqLen + 1; + if (par.filterSeqDBFile.empty() == false) { + // Stock sizes its aligner buffers with the larger of the two databases + // (Align2clust.cpp:502), because the filter gate compares against + // sequences drawn from the full database, not the pass-2 one. + const DenseIndex::Info full = DenseIndex::readInfo(par.filterSeqDBFile); + maxSeqLen = std::max(maxSeqLen, full.maxSeqLen + 1); + } // Same x-drop as stock (Align2clust.cpp:419, MIN_SIZE 32). const int32_t xDrop = 32 * par.gapExtend.values.aminoacid() + par.gapOpen.values.aminoacid(); + const unsigned int swMode = Alignment::initSWMode(par.alignmentMode, par.covThr, par.seqIdThr); + + // Pass 2 only: stock's --filter-cludb-file gate. + FilterGate *gate = NULL; + if (par.filterCluDBFile.empty() == false) { + if (par.filterSeqDBFile.empty() || par.keyMapFile.empty()) { + Debug(Debug::ERROR) << "--filter-cludb-file needs --filter-seqdb-file and --key-map\n"; + EXIT(EXIT_FAILURE); + } + gate = new FilterGate(par.filterSeqDBFile); + gate->load(par.keyMapFile, par.filterCluDBFile); + Debug(Debug::INFO) << "Filter gate: " << gate->keymap.size() << " representatives, " + << gate->members.size() << " pass-1 members\n"; + } Debug(Debug::INFO) << "Aligning " << bucketCount << " edge buckets; score-per-column cutoff " << scorePerColThreshold << "\n"; @@ -244,6 +405,22 @@ int alignparallel(int argc, const char **argv, const Command &command) { needed.erase(std::unique(needed.begin(), needed.end()), needed.end()); sequences.load(needed); + // The gate compares against the pass-1 cluster members of each target, + // which live in the *full* database; fetch exactly those, in key order. + if (gate != NULL) { + std::vector gateKeys; + for (size_t i = 0; i < edges.size(); i++) { + const uint64_t sub = edges[i].getMember(); + if (sub >= gate->keymap.size()) continue; + for (uint64_t j = gate->clusterStart[sub]; j < gate->clusterStart[sub + 1]; j++) { + gateKeys.push_back(gate->members[j]); + } + } + SORT_PARALLEL(gateKeys.begin(), gateKeys.end()); + gateKeys.erase(std::unique(gateKeys.begin(), gateKeys.end()), gateKeys.end()); + gate->fullSeqs.load(gateKeys); + } + // Edges are sorted by (rep, member), so a representative's edges are // contiguous and its query profile is built once. std::vector repStarts; @@ -265,6 +442,12 @@ int alignparallel(int argc, const char **argv, const Command &command) { &evaluer, par.compBiasCorrection, par.compBiasCorrectionScale, -par.gapOpen.values.aminoacid(), -par.gapExtend.values.aminoacid()); + Matcher matcher(Parameters::DBTYPE_AMINO_ACIDS, maxSeqLen, subMat, &evaluer, + par.compBiasCorrection, par.compBiasCorrectionScale, + par.gapOpen.values.aminoacid(), par.gapExtend.values.aminoacid(), + 0.0, par.zdrop); + Sequence element(maxSeqLen, Parameters::DBTYPE_AMINO_ACIDS, subMat, 0, false, + par.compBiasCorrection); #pragma omp for schedule(dynamic, 16) for (size_t g = 0; g < repStarts.size() - 1; g++) { @@ -277,6 +460,7 @@ int alignparallel(int argc, const char **argv, const Command &command) { } query.mapSequence(0, static_cast(edges[from].getRep()), repSeq, repLen); aligner.initQuery(&query); + matcher.initQuery(&query); for (size_t e = from; e < to; e++) { unsigned int memberLen = 0; @@ -319,6 +503,10 @@ int alignparallel(int argc, const char **argv, const Command &command) { seqId >= (par.seqIdThr - std::numeric_limits::epsilon()); if (hasAlnLen && hasCoverage && hasSeqId && hasEvalue) { + if (passesFilterGate(gate, edges[e].getMember(), query, element, + aligner, matcher, par, swMode) == false) { + continue; + } // The greedy ranks by alignment score, not k-mer count. edges[e].score = static_cast(std::min(aln.bitScore, 255)); @@ -374,6 +562,10 @@ int alignparallel(int argc, const char **argv, const Command &command) { // the reduce, so a representative is never its own member. if (Alignment::checkCriteria(result, false, par.evalThr, par.seqIdThr, par.alnLenThr, par.covMode, par.covThr)) { + if (passesFilterGate(gate, edges[e].getMember(), query, element, + aligner, matcher, par, swMode) == false) { + continue; + } edges[e].score = static_cast(std::min(gapped.score1, 255)); survives[e] = 1; diff --git a/src/linclust/createrepdb.cpp b/src/linclust/createrepdb.cpp new file mode 100644 index 000000000..41ac171b9 --- /dev/null +++ b/src/linclust/createrepdb.cpp @@ -0,0 +1,248 @@ +/* + * createrepdb -- builds the representative sub-database for the next linclust pass. + * + * Replaces `createsubdb` for this pipeline, for two reasons. + * + * The first is the usual one: `createsubdb` opens the database with USE_INDEX and + * so holds the text index resident, 24 bytes per sequence -- 2.4 TB at 1e11. This + * streams through the dense companion index instead and holds one bit per key. + * + * The second is specific and easy to miss: **a sub-database of representatives has + * sparse keys**, and every stage of the distributed pipeline assumes dense ones -- + * DenseIndex treats a key as an array offset, the greedy sweep walks the key space + * in order, and the edge and merge buckets are key ranges. Feeding a `createsubdb` + * output into pass 2 would break all three. So the representatives are *re-keyed* + * densely here, and a `subkey -> original key` map is written alongside so the + * pass-2 clustering can be translated back before merging. + * + * Re-keying is nearly free because of how the keys were assigned in the first + * place. Original keys are length-ranked, so any subset of them is *already* in + * descending length order; the i-th representative in ascending original-key order + * is exactly sub-key i. No sort is needed, and the new database is a sequential + * copy rather than a gather. + */ +#include "Command.h" +#include "Debug.h" +#include "DenseIndex.h" +#include "FileUtil.h" +#include "Parameters.h" +#include "Util.h" + +#include +#include +#include +#include +#include + +#include +#include + +#ifdef OPENMP +#include +#endif + +namespace { + +class Bitmap { +public: + explicit Bitmap(uint64_t bits) : words((bits + 63) / 64, 0) {} + bool get(uint64_t i) const { return (words[i >> 6] >> (i & 63)) & 1ULL; } + void set(uint64_t i) { words[i >> 6] |= 1ULL << (i & 63); } + uint64_t bytes() const { return words.size() * sizeof(uint64_t); } + +private: + std::vector words; +}; + +int openOrDie(const std::string &path, int flags) { + const int fd = open(path.c_str(), flags, 0666); + if (fd < 0) { + Debug(Debug::ERROR) << "Cannot open " << path << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + return fd; +} + +void readAt(int fd, void *dst, size_t len, size_t off, const char *what) { + char *p = static_cast(dst); + size_t done = 0; + while (done < len) { + const ssize_t got = pread(fd, p + done, len - done, static_cast(off + done)); + if (got <= 0) { + if (got < 0 && errno == EINTR) continue; + Debug(Debug::ERROR) << "Cannot read " << what << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + done += static_cast(got); + } +} + +void writeAt(int fd, const void *src, size_t len, size_t off, const char *what) { + const char *p = static_cast(src); + size_t done = 0; + while (done < len) { + const ssize_t put = pwrite(fd, p + done, len - done, static_cast(off + done)); + if (put <= 0) { + if (put < 0 && errno == EINTR) continue; + Debug(Debug::ERROR) << "Cannot write " << what << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + done += static_cast(put); + } +} + +void allocate(const std::string &path, uint64_t size) { + const int fd = openOrDie(path, O_WRONLY | O_CREAT | O_TRUNC); + if (size > 0 && ftruncate(fd, static_cast(size)) != 0) { + Debug(Debug::ERROR) << "Cannot size " << path << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + close(fd); +} + +// Copies the entries flagged in `keep` into a new dense database, preserving key +// order. Returns the total data bytes and the longest entry. +struct CopyResult { + uint64_t dataBytes; + uint32_t maxLen; +}; + +CopyResult copyFlagged(const std::string &srcDb, const std::string &dstDb, const Bitmap &keep, + uint64_t entryCount, uint64_t keptCount, int threads, + std::vector *keyMap) { + const int srcIdx = openOrDie(DenseIndex::fileName(srcDb), O_RDONLY); + const int srcData = openOrDie(srcDb, O_RDONLY); + + // Pass 1: per-key kept lengths, so the destination offsets are a prefix sum. + // The index is read sequentially in blocks rather than randomly per key. + std::vector offsets(keptCount + 1, 0); + std::vector lengths(keptCount, 0); + std::vector srcEntries(keptCount); + { + const uint64_t block = 1 << 20; + std::vector buf(block); + uint64_t out = 0; + for (uint64_t from = 0; from < entryCount; from += block) { + const uint64_t n = std::min(block, entryCount - from); + readAt(srcIdx, buf.data(), n * sizeof(DenseIndex::Entry), + DenseIndex::entryOffset(from), "source index"); + for (uint64_t i = 0; i < n; i++) { + if (keep.get(from + i) == false) continue; + srcEntries[out] = buf[i]; + lengths[out] = buf[i].length; + if (keyMap != NULL) (*keyMap)[out] = from + i; + out++; + } + } + } + CopyResult result; + result.maxLen = 0; + uint64_t total = 0; + for (uint64_t i = 0; i < keptCount; i++) { + offsets[i] = total; + total += lengths[i]; + result.maxLen = std::max(result.maxLen, lengths[i]); + } + offsets[keptCount] = total; + result.dataBytes = total; + + allocate(dstDb, total); + DenseIndex::createEmpty(dstDb, keptCount, 0, total, result.maxLen); + + const int dstData = openOrDie(dstDb, O_WRONLY); + const int dstIdx = openOrDie(DenseIndex::fileName(dstDb), O_WRONLY); + + // Pass 2: copy. Entries are independent and both sides are in ascending offset + // order, so this is a threaded forward scan rather than a gather. +#pragma omp parallel num_threads(threads) + { + std::vector buf; + std::vector idxBuf; + const uint64_t chunk = 4096; +#pragma omp for schedule(dynamic, 1) + for (uint64_t start = 0; start < keptCount; start += chunk) { + const uint64_t stop = std::min(start + chunk, keptCount); + const uint64_t bytes = offsets[stop] - offsets[start]; + buf.resize(static_cast(bytes)); + idxBuf.resize(static_cast(stop - start)); + for (uint64_t i = start; i < stop; i++) { + readAt(srcData, buf.data() + (offsets[i] - offsets[start]), lengths[i], + srcEntries[i].offset, "source data"); + idxBuf[i - start].offset = offsets[i]; + idxBuf[i - start].length = lengths[i]; + } + writeAt(dstData, buf.data(), static_cast(bytes), offsets[start], "database"); + writeAt(dstIdx, idxBuf.data(), idxBuf.size() * sizeof(DenseIndex::Entry), + DenseIndex::entryOffset(start), "index"); + } + } + + close(srcIdx); + close(srcData); + close(dstData); + close(dstIdx); + return result; +} + +} // namespace + +int createrepdb(int argc, const char **argv, const Command &command) { + Parameters &par = Parameters::getInstance(); + par.parseParameters(argc, argv, command, true, 0, 0); + + const std::string seqDb = par.db1; + const std::string clusterTsv = par.db2; + const std::string repDb = par.db3; + + const DenseIndex::Info info = DenseIndex::readInfo(seqDb); + + // A key is a representative exactly when it appears in the first column. + Bitmap isRep(info.entryCount); + uint64_t repCount = 0; + { + FILE *f = FileUtil::openFileOrDie(clusterTsv.c_str(), "r", true); + char *line = NULL; + size_t cap = 0; + while (getline(&line, &cap, f) > 0) { + const uint64_t rep = strtoull(line, NULL, 10); + if (rep < info.entryCount && isRep.get(rep) == false) { + isRep.set(rep); + repCount++; + } + } + free(line); + fclose(f); + } + Debug(Debug::INFO) << repCount << " representatives of " << info.entryCount << " sequences (" + << isRep.bytes() / (1024 * 1024) << " MB of flags)\n"; + if (repCount == 0) { + Debug(Debug::ERROR) << "No representatives found in " << clusterTsv << "\n"; + EXIT(EXIT_FAILURE); + } + + std::vector keyMap(repCount, 0); + const CopyResult seq = copyFlagged(seqDb, repDb, isRep, info.entryCount, repCount, par.threads, + &keyMap); + copyFlagged(seqDb + "_h", repDb + "_h", isRep, info.entryCount, repCount, par.threads, NULL); + + // subkey -> original key, dense in the sub-key space and in ascending order, + // so the translation back is a sequential read rather than a lookup structure. + const std::string mapFile = repDb + ".keymap"; + FILE *m = FileUtil::openAndDelete(mapFile.c_str(), "wb"); + if (fwrite(keyMap.data(), sizeof(uint64_t), keyMap.size(), m) != keyMap.size()) { + Debug(Debug::ERROR) << "Cannot write " << mapFile << "\n"; + EXIT(EXIT_FAILURE); + } + fclose(m); + + const int dbType = FileUtil::parseDbType(seqDb.c_str()); + FileUtil::writeFile(repDb + ".dbtype", reinterpret_cast(&dbType), + sizeof(int)); + const int hdrType = Parameters::DBTYPE_GENERIC_DB; + FileUtil::writeFile(repDb + "_h.dbtype", reinterpret_cast(&hdrType), + sizeof(int)); + + Debug(Debug::INFO) << "Wrote " << repDb << ": " << repCount << " sequences, " << seq.dataBytes + << " data bytes, longest " << seq.maxLen << "\n"; + return EXIT_SUCCESS; +} diff --git a/src/linclust/kmermatcherparallel.cpp b/src/linclust/kmermatcherparallel.cpp index de4643408..833a0c654 100644 --- a/src/linclust/kmermatcherparallel.cpp +++ b/src/linclust/kmermatcherparallel.cpp @@ -254,11 +254,35 @@ int kmermatcherparallel(int argc, const char **argv, const Command &command) { Debug(Debug::INFO) << "K-mer shuffle: " << sizing.partitionCount << " partitions, " << sizing.waveCount << " wave(s), " << sizing.totalKmerBytes << " k-mer bytes, " << sizing.bytesPerPartition << " bytes per partition\n"; + // Which slice of the partition space this invocation writes. Waves exist + // because the whole shuffle need not fit at once: each re-extracts every + // k-mer but keeps only its own slice, so peak scratch is divided by the wave + // count at the cost of that many extraction passes. + unsigned int waveFrom = 0; + unsigned int waveTo = sizing.partitionCount; if (sizing.waveCount > 1) { - Debug(Debug::ERROR) << "The scratch budget of " << par.scratchBudget << " bytes does not " - << "hold the whole k-mer shuffle; it would need " << sizing.waveCount - << " waves, each reduced and deleted before the next is extracted. " - << "Waves are not implemented yet -- raise --scratch-budget.\n"; + if (par.kmerWave < 0) { + Debug(Debug::ERROR) << "The scratch budget of " << par.scratchBudget << " bytes needs " + << sizing.waveCount << " extraction waves. Run waves 0.." + << (sizing.waveCount - 1) << " with --kmer-wave, reducing and " + << "deleting each wave's buckets before starting the next, or " + << "raise --scratch-budget to fit a single wave.\n"; + EXIT(EXIT_FAILURE); + } + if (static_cast(par.kmerWave) >= sizing.waveCount) { + Debug(Debug::ERROR) << "--kmer-wave " << par.kmerWave << " is out of range; this run " + << "has " << sizing.waveCount << " waves\n"; + EXIT(EXIT_FAILURE); + } + // Exact: the sizing guarantees the wave count divides P. + const unsigned int perWave = sizing.partitionCount / sizing.waveCount; + waveFrom = static_cast(par.kmerWave) * perWave; + waveTo = waveFrom + perWave; + Debug(Debug::INFO) << "Wave " << par.kmerWave << " of " << sizing.waveCount + << ": partitions [" << waveFrom << ", " << waveTo << ")\n"; + } else if (par.kmerWave > 0) { + Debug(Debug::ERROR) << "--kmer-wave " << par.kmerWave << " given but this run needs only " + << "one wave\n"; EXIT(EXIT_FAILURE); } @@ -301,10 +325,12 @@ int kmermatcherparallel(int argc, const char **argv, const Command &command) { KmerPartitioner partitioner(sizing.partitionCount); // One writer for the whole process, shared by every thread and kept open // across items so bucket files are appended to rather than reopened. - KmerBucketWriter writer(kmerDir, sizing.partitionCount, "w" + SSTR(workerId)); + KmerBucketWriter writer(kmerDir, sizing.partitionCount, "w" + SSTR(workerId), + 1024 * 1024 * 1024, waveFrom, waveTo); { - WorkQueue queue(coordDir + "/scan.queue", itemCount); + WorkQueue queue(coordDir + "/scan." + SSTR(par.kmerWave < 0 ? 0 : par.kmerWave) + + ".queue", itemCount); // Claimed one item at a time by the process rather than by each thread: // the extraction inside an item is already threaded, and nesting a second // parallel region inside a claiming one would oversubscribe the node. diff --git a/src/linclust/kmerreduceparallel.cpp b/src/linclust/kmerreduceparallel.cpp index 7de4a2f0b..ae65c592a 100644 --- a/src/linclust/kmerreduceparallel.cpp +++ b/src/linclust/kmerreduceparallel.cpp @@ -41,12 +41,15 @@ #include #endif +#include #include #include #include #include #include +#include + namespace { BaseMatrix *createSubstitutionMatrix(Parameters &par, int dbType) { @@ -358,6 +361,7 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { } unsigned int partitionCount = 0; unsigned int kmerSize = 0; + unsigned int waveCount = 1; { FILE *file = FileUtil::openFileOrDie(manifestPath.c_str(), "r", true); char name[64]; @@ -368,6 +372,8 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { partitionCount = static_cast(value); } else if (key == "kmerSize") { kmerSize = static_cast(value); + } else if (key == "waveCount") { + waveCount = static_cast(value); } } fclose(file); @@ -376,6 +382,35 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { Debug(Debug::ERROR) << "Shuffle manifest " << manifestPath << " has no partition count\n"; EXIT(EXIT_FAILURE); } + // A wave's map wrote only its own slice of partition space, so its reduce + // claims exactly that slice. The slicing must be identical to the map's, and + // each wave needs its own queue: a shared one would record the whole + // partition space done after wave 0 and skip every later wave outright. + unsigned int waveFrom = 0; + unsigned int waveTo = partitionCount; + if (waveCount > 1) { + if (par.kmerWave < 0) { + Debug(Debug::ERROR) << "This shuffle was written in " << waveCount + << " waves. Reduce each one with --kmer-wave 0.." + << (waveCount - 1) << ", matching the wave its map wrote.\n"; + EXIT(EXIT_FAILURE); + } + if (static_cast(par.kmerWave) >= waveCount) { + Debug(Debug::ERROR) << "--kmer-wave " << par.kmerWave << " is out of range; this " + << "shuffle has " << waveCount << " waves\n"; + EXIT(EXIT_FAILURE); + } + // Exact: the sizing guarantees the wave count divides P. + const unsigned int perWave = partitionCount / waveCount; + waveFrom = static_cast(par.kmerWave) * perWave; + waveTo = waveFrom + perWave; + Debug(Debug::INFO) << "Wave " << par.kmerWave << " of " << waveCount << ": partitions [" + << waveFrom << ", " << waveTo << ")\n"; + } else if (par.kmerWave > 0) { + Debug(Debug::ERROR) << "--kmer-wave " << par.kmerWave << " given but this shuffle was " + << "written in one wave\n"; + EXIT(EXIT_FAILURE); + } // The map decided k, so take it from the manifest rather than re-deriving it // here: the two must agree, and the map's value is the one on disk. par.kmerSize = static_cast(kmerSize); @@ -428,10 +463,13 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { uint64_t edgeCount = 0; { - WorkQueue queue(reduceCoordDir + "/reduce.queue", static_cast(partitionCount)); + WorkQueue queue(reduceCoordDir + "/reduce." + SSTR(par.kmerWave < 0 ? 0 : par.kmerWave) + + ".queue", + static_cast(waveTo - waveFrom)); // One partition at a time per process: the sort and the greedy inside a // partition are already threaded, and a partition is sized to fill a node. - const bool finished = queue.drain(workerId, [&](size_t partition) { + const bool finished = queue.drain(workerId, [&](size_t item) { + const size_t partition = waveFrom + item; if (info.maxSeqLen < SHRT_MAX) { edgeCount += reducePartition(kmerDir, static_cast(partition), dbType, par, subMat, *edgeWriter, bucketSpan); @@ -449,6 +487,26 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { } } + if (waveCount > 1) { + // Waves exist only because scratch cannot hold the whole shuffle at once, + // so a reduced wave's buckets have to go before the next wave's map runs. + // Safe here because drain() returns only once every partition of the wave + // is recorded done, which means no worker is still reading one. Several + // workers reach this together, so a file another already unlinked is not + // an error. + for (unsigned int p = waveFrom; p < waveTo; p++) { + const std::vector shards = KmerBucketReader::shardFiles(kmerDir, p); + for (size_t i = 0; i < shards.size(); i++) { + if (unlink(shards[i].c_str()) != 0 && errno != ENOENT) { + Debug(Debug::ERROR) << "Cannot remove reduced bucket " << shards[i] << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + } + } + Debug(Debug::INFO) << "Removed the k-mer buckets of wave " << par.kmerWave << "\n"; + } + Debug(Debug::INFO) << "Worker " << workerId << " wrote " << edgeCount << " candidate edges\n"; edgeWriter->close(); diff --git a/src/linclust/mergeclusterparallel.cpp b/src/linclust/mergeclusterparallel.cpp new file mode 100644 index 000000000..e8cb217a7 --- /dev/null +++ b/src/linclust/mergeclusterparallel.cpp @@ -0,0 +1,273 @@ +/* + * mergeclusterparallel -- composes the clusterings of successive linclust passes. + * + * linclust clusters, then re-clusters the representatives, then folds the second + * result into the first: a sequence assigned to representative r in pass 1 ends + * up under whatever representative r was given in pass 2. Stock does this in + * `mergeclusters` with `std::list[dbSize]` (mergeclusters.cpp:28) and + * splices lists between entries. That array is 24 bytes per sequence of *empty + * list headers* before a single member is stored -- 2.4 TB at 1e11 and 24 TB at + * 1e12 -- and every member is held in memory besides. + * + * The composition is really just a join, `final(x) = later(earlier(x))`, so with + * dense keys it needs no resident per-key state at all: + * + * 1. radix-partition both clusterings into key-range buckets -- the later one by + * the member it reassigns, the earlier one by its representative, so the two + * sides of the join land in the same bucket; + * 2. per bucket, the later clustering's mapping is a *dense array over that key + * range*, because the keys in a range are contiguous integers; + * 3. stream the earlier clustering's bucket through that array and emit. + * + * Peak memory is one bucket's remap array, which the bucket count sets, rather + * than anything proportional to the database. + */ +#include "Command.h" +#include "Debug.h" +#include "DenseIndex.h" +#include "FileUtil.h" +#include "Parameters.h" +#include "Util.h" + +#include +#include +#include +#include +#include + +namespace { + +// (key, value) as written to a bucket: for the later clustering the key is the +// member being reassigned, for the earlier one it is the representative. +struct __attribute__((__packed__)) Pair { + uint64_t key; + uint64_t value; +}; + +const uint64_t INVALID = UINT64_MAX; + +class BucketWriter { +public: + BucketWriter(const std::string &prefix, unsigned int buckets, size_t bufferPairs = 1 << 16) + : prefix(prefix), buckets(buckets), bufferPairs(bufferPairs), closed(false) { + files.assign(buckets, NULL); + this->buffers.resize(buckets); + } + ~BucketWriter() { close(); } + + void append(unsigned int bucket, uint64_t key, uint64_t value) { + Pair p; + p.key = key; + p.value = value; + buffers[bucket].push_back(p); + if (buffers[bucket].size() >= bufferPairs) { + flush(bucket); + } + } + + void close() { + if (closed) { + return; + } + closed = true; + for (unsigned int b = 0; b < buckets; b++) { + flush(b); + if (files[b] != NULL) { + fclose(files[b]); + files[b] = NULL; + } + } + } + + static std::string path(const std::string &prefix, unsigned int bucket) { + return prefix + "." + SSTR(bucket); + } + +private: + void flush(unsigned int bucket) { + if (buffers[bucket].empty()) { + return; + } + if (files[bucket] == NULL) { + files[bucket] = fopen(path(prefix, bucket).c_str(), "wb"); + if (files[bucket] == NULL) { + Debug(Debug::ERROR) << "Cannot open " << path(prefix, bucket) << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + } + if (fwrite(buffers[bucket].data(), sizeof(Pair), buffers[bucket].size(), files[bucket]) != + buffers[bucket].size()) { + Debug(Debug::ERROR) << "Cannot write bucket " << bucket << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + buffers[bucket].clear(); + } + + std::string prefix; + unsigned int buckets; + size_t bufferPairs; + std::vector > buffers; + std::vector files; + bool closed; +}; + +// Streams a "repmember" TSV, bucketing on whichever column the join needs. +void partition(const std::string &tsv, BucketWriter &writer, uint64_t bucketSpan, bool byRep) { + FILE *file = FileUtil::openFileOrDie(tsv.c_str(), "r", true); + char *line = NULL; + size_t cap = 0; + ssize_t len; + while ((len = getline(&line, &cap, file)) > 0) { + char *tab = strchr(line, '\t'); + if (tab == NULL) { + continue; + } + const uint64_t rep = strtoull(line, NULL, 10); + const uint64_t member = strtoull(tab + 1, NULL, 10); + const uint64_t key = byRep ? rep : member; + const uint64_t value = byRep ? member : rep; + writer.append(static_cast(key / bucketSpan), key, value); + } + free(line); + fclose(file); +} + +std::vector readBucket(const std::string &prefix, unsigned int bucket) { + std::vector out; + const std::string p = BucketWriter::path(prefix, bucket); + if (FileUtil::fileExists(p.c_str()) == false) { + return out; + } + const size_t bytes = FileUtil::getFileSize(p); + if (bytes == 0 || bytes % sizeof(Pair) != 0) { + return out; + } + out.resize(bytes / sizeof(Pair)); + FILE *f = FileUtil::openFileOrDie(p.c_str(), "rb", true); + if (fread(out.data(), sizeof(Pair), out.size(), f) != out.size()) { + Debug(Debug::ERROR) << "Cannot read " << p << "\n"; + EXIT(EXIT_FAILURE); + } + fclose(f); + return out; +} + +inline void appendPair(std::string &out, uint64_t a, uint64_t b) { + char tmp[48]; + char digits[24]; + int n = 0, d = 0; + do { digits[d++] = static_cast('0' + (a % 10)); a /= 10; } while (a); + while (d) tmp[n++] = digits[--d]; + tmp[n++] = '\t'; + do { digits[d++] = static_cast('0' + (b % 10)); b /= 10; } while (b); + while (d) tmp[n++] = digits[--d]; + tmp[n++] = '\n'; + out.append(tmp, n); +} + +// earlier: member -> representative. later: that representative -> its new one. +void compose(const std::string &earlier, const std::string &later, const std::string &out, + uint64_t entryCount, const std::string &tmpPrefix, uint64_t bucketSpan, + unsigned int buckets) { + { + BucketWriter laterW(tmpPrefix + ".later", buckets); + BucketWriter earlierW(tmpPrefix + ".earlier", buckets); + // Bucket the later clustering by the member it reassigns, and the earlier + // one by its representative: those are the two sides of the join, so they + // meet in the same bucket. + partition(later, laterW, bucketSpan, false); + partition(earlier, earlierW, bucketSpan, true); + } + + FILE *result = FileUtil::openAndDelete(out.c_str(), "w"); + std::string buffer; + buffer.reserve(64 * 1024 * 1024); + std::vector remap; + + for (unsigned int b = 0; b < buckets; b++) { + const uint64_t lo = b * bucketSpan; + if (lo >= entryCount) { + break; + } + const uint64_t hi = std::min(lo + bucketSpan, entryCount); + + // Dense over the key range, which is what dense keys buy: no hash table, + // no per-key state outside this bucket. + remap.assign(static_cast(hi - lo), INVALID); + const std::vector laterPairs = readBucket(tmpPrefix + ".later", b); + for (size_t i = 0; i < laterPairs.size(); i++) { + remap[static_cast(laterPairs[i].key - lo)] = laterPairs[i].value; + } + + const std::vector earlierPairs = readBucket(tmpPrefix + ".earlier", b); + for (size_t i = 0; i < earlierPairs.size(); i++) { + const uint64_t rep = earlierPairs[i].key; + const uint64_t mapped = remap[static_cast(rep - lo)]; + appendPair(buffer, mapped == INVALID ? rep : mapped, earlierPairs[i].value); + if (buffer.size() > 32 * 1024 * 1024) { + fwrite(buffer.data(), 1, buffer.size(), result); + buffer.clear(); + } + } + FileUtil::remove(BucketWriter::path(tmpPrefix + ".later", b).c_str()); + FileUtil::remove(BucketWriter::path(tmpPrefix + ".earlier", b).c_str()); + } + if (buffer.empty() == false) { + fwrite(buffer.data(), 1, buffer.size(), result); + } + if (fclose(result) != 0) { + Debug(Debug::ERROR) << "Cannot close " << out << "\n"; + EXIT(EXIT_FAILURE); + } +} + +} // namespace + +int mergeclusterparallel(int argc, const char **argv, const Command &command) { + Parameters &par = Parameters::getInstance(); + par.parseParameters(argc, argv, command, false, Parameters::PARSE_VARIADIC, 0); + + std::vector files(par.filenames); + const std::string seqDb = files.front(); + files.erase(files.begin()); + const std::string out = files.back(); + files.pop_back(); + if (files.size() < 2) { + Debug(Debug::ERROR) << "Need at least two clusterings to merge\n"; + EXIT(EXIT_FAILURE); + } + par.printParameters(command.cmd, argc, argv, *command.params); + + const DenseIndex::Info info = DenseIndex::readInfo(seqDb); + + // One bucket's remap array is 8 bytes per key in its range; size the buckets so + // that stays a modest slice of the memory limit. + // A floor only so a tiny limit does not explode into pathologically many + // buckets; low enough that the multi-bucket path is reachable in testing, + // which matters because that is the path used at scale. + const uint64_t targetBytes = std::max(Util::computeMemory(par.splitMemoryLimit) / 8, + 1ULL * 1024 * 1024); + unsigned int buckets = 1; + while (buckets < 65536 && (info.entryCount / buckets) * sizeof(uint64_t) > targetBytes) { + buckets *= 2; + } + const uint64_t bucketSpan = (info.entryCount + buckets - 1) / buckets; + Debug(Debug::INFO) << "Merging " << files.size() << " clusterings over " << buckets + << " key buckets of " << bucketSpan << " keys\n"; + + // Fold left: the accumulated clustering is always the "earlier" side. + std::string current = files[0]; + std::string scratch = out + ".tmp"; + for (size_t i = 1; i < files.size(); i++) { + const std::string target = (i + 1 == files.size()) ? out : (scratch + ".step" + SSTR(i)); + compose(current, files[i], target, info.entryCount, out + ".part", bucketSpan, buckets); + if (i > 1) { + FileUtil::remove(current.c_str()); + } + current = target; + } + + Debug(Debug::INFO) << "Wrote " << out << "\n"; + return EXIT_SUCCESS; +} diff --git a/src/linclust/translatecluster.cpp b/src/linclust/translatecluster.cpp new file mode 100644 index 000000000..4a1c3a7b1 --- /dev/null +++ b/src/linclust/translatecluster.cpp @@ -0,0 +1,243 @@ +/* + * translatecluster -- rewrites a clustering from sub-key space into original keys. + * + * `createrepdb` re-keys the representatives densely so the next linclust pass can + * run on them, which means that pass's clustering comes back in sub-key space and + * has to be translated before it can be merged with the first pass. + * + * The map is dense and ascending in sub-key space, so translating one column is a + * sequential read. The difficulty is that a clustering has *two* key columns and + * only one of them can be in order at a time. Holding the whole map resident would + * be 8 bytes per representative -- ~296 GB at 1e11 given the 37% representative + * fraction measured on the 5B run, and ~3 TB at 1e12, which does not fit. + * + * So each column is translated in its own bucketed pass: partition by the sub-key + * being translated, then per bucket read exactly that contiguous slice of the map. + * Peak memory is one slice, and every read of the map is sequential. + */ +#include "Command.h" +#include "Debug.h" +#include "FileUtil.h" +#include "Parameters.h" +#include "Util.h" + +#include +#include +#include +#include +#include + +#include +#include + +namespace { + +struct __attribute__((__packed__)) Pair { + uint64_t key; // the sub-key being translated in this pass + uint64_t other; // the column carried through untouched +}; + +class Buckets { +public: + Buckets(const std::string &prefix, unsigned int count, size_t bufferPairs = 1 << 16) + : prefix(prefix), count(count), bufferPairs(bufferPairs), closed(false) { + files.assign(count, NULL); + buffers.resize(count); + } + ~Buckets() { close(); } + + void append(unsigned int b, uint64_t key, uint64_t other) { + Pair p; + p.key = key; + p.other = other; + buffers[b].push_back(p); + if (buffers[b].size() >= bufferPairs) flush(b); + } + + void close() { + if (closed) return; + closed = true; + for (unsigned int b = 0; b < count; b++) { + flush(b); + if (files[b] != NULL) { fclose(files[b]); files[b] = NULL; } + } + } + + static std::string path(const std::string &prefix, unsigned int b) { + return prefix + "." + SSTR(b); + } + + static std::vector read(const std::string &prefix, unsigned int b) { + std::vector out; + const std::string p = path(prefix, b); + if (FileUtil::fileExists(p.c_str()) == false) return out; + const size_t bytes = FileUtil::getFileSize(p); + if (bytes == 0 || bytes % sizeof(Pair) != 0) return out; + out.resize(bytes / sizeof(Pair)); + FILE *f = FileUtil::openFileOrDie(p.c_str(), "rb", true); + if (fread(out.data(), sizeof(Pair), out.size(), f) != out.size()) { + Debug(Debug::ERROR) << "Cannot read " << p << "\n"; + EXIT(EXIT_FAILURE); + } + fclose(f); + return out; + } + +private: + void flush(unsigned int b) { + if (buffers[b].empty()) return; + if (files[b] == NULL) { + files[b] = fopen(path(prefix, b).c_str(), "wb"); + if (files[b] == NULL) { + Debug(Debug::ERROR) << "Cannot open " << path(prefix, b) << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + } + if (fwrite(buffers[b].data(), sizeof(Pair), buffers[b].size(), files[b]) != + buffers[b].size()) { + Debug(Debug::ERROR) << "Cannot write bucket " << b << "\n"; + EXIT(EXIT_FAILURE); + } + buffers[b].clear(); + } + + std::string prefix; + unsigned int count; + size_t bufferPairs; + std::vector > buffers; + std::vector files; + bool closed; +}; + +// Reads map[lo, hi) -- one contiguous slice, never the whole file. +std::vector mapSlice(int fd, uint64_t lo, uint64_t hi) { + std::vector out(static_cast(hi - lo)); + size_t want = out.size() * sizeof(uint64_t), done = 0; + char *p = reinterpret_cast(out.data()); + while (done < want) { + const ssize_t got = pread(fd, p + done, want - done, + static_cast(lo * sizeof(uint64_t) + done)); + if (got <= 0) { + if (got < 0 && errno == EINTR) continue; + Debug(Debug::ERROR) << "Cannot read key map: " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + done += static_cast(got); + } + return out; +} + +inline void appendPair(std::string &out, uint64_t a, uint64_t b) { + char tmp[48], digits[24]; + int n = 0, d = 0; + do { digits[d++] = static_cast('0' + (a % 10)); a /= 10; } while (a); + while (d) tmp[n++] = digits[--d]; + tmp[n++] = '\t'; + do { digits[d++] = static_cast('0' + (b % 10)); b /= 10; } while (b); + while (d) tmp[n++] = digits[--d]; + tmp[n++] = '\n'; + out.append(tmp, n); +} + +} // namespace + +int translatecluster(int argc, const char **argv, const Command &command) { + Parameters &par = Parameters::getInstance(); + par.parseParameters(argc, argv, command, true, 0, 0); + + const std::string inTsv = par.db1; + const std::string mapFile = par.db2; + const std::string outTsv = par.db3; + + const size_t mapBytes = FileUtil::getFileSize(mapFile); + if (mapBytes % sizeof(uint64_t) != 0) { + Debug(Debug::ERROR) << mapFile << " is not a whole number of keys\n"; + EXIT(EXIT_FAILURE); + } + const uint64_t subCount = mapBytes / sizeof(uint64_t); + + const uint64_t targetBytes = std::max(Util::computeMemory(par.splitMemoryLimit) / 8, + 1ULL * 1024 * 1024); + unsigned int buckets = 1; + while (buckets < 65536 && (subCount / buckets) * sizeof(uint64_t) > targetBytes) { + buckets *= 2; + } + const uint64_t span = (subCount + buckets - 1) / buckets; + Debug(Debug::INFO) << "Translating " << subCount << " sub-keys over " << buckets + << " buckets of " << span << "\n"; + + const int mapFd = open(mapFile.c_str(), O_RDONLY); + if (mapFd < 0) { + Debug(Debug::ERROR) << "Cannot open " << mapFile << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + + const std::string tmpA = outTsv + ".bymember"; + const std::string tmpB = outTsv + ".byrep"; + + // Pass 1: bucket by the member sub-key, then translate it. + { + Buckets byMember(tmpA, buckets); + FILE *f = FileUtil::openFileOrDie(inTsv.c_str(), "r", true); + char *line = NULL; + size_t cap = 0; + while (getline(&line, &cap, f) > 0) { + char *tab = strchr(line, '\t'); + if (tab == NULL) continue; + const uint64_t subRep = strtoull(line, NULL, 10); + const uint64_t subMember = strtoull(tab + 1, NULL, 10); + byMember.append(static_cast(subMember / span), subMember, subRep); + } + free(line); + fclose(f); + } + { + Buckets byRep(tmpB, buckets); + for (unsigned int b = 0; b < buckets; b++) { + const uint64_t lo = b * span; + if (lo >= subCount) break; + const uint64_t hi = std::min(lo + span, subCount); + const std::vector slice = mapSlice(mapFd, lo, hi); + const std::vector pairs = Buckets::read(tmpA, b); + for (size_t i = 0; i < pairs.size(); i++) { + const uint64_t origMember = slice[static_cast(pairs[i].key - lo)]; + // Re-bucket on the representative sub-key for the second pass. + byRep.append(static_cast(pairs[i].other / span), pairs[i].other, + origMember); + } + FileUtil::remove(Buckets::path(tmpA, b).c_str()); + } + } + + // Pass 2: translate the representative sub-key. + FILE *out = FileUtil::openAndDelete(outTsv.c_str(), "w"); + std::string buffer; + buffer.reserve(64 * 1024 * 1024); + uint64_t written = 0; + for (unsigned int b = 0; b < buckets; b++) { + const uint64_t lo = b * span; + if (lo >= subCount) break; + const uint64_t hi = std::min(lo + span, subCount); + const std::vector slice = mapSlice(mapFd, lo, hi); + const std::vector pairs = Buckets::read(tmpB, b); + for (size_t i = 0; i < pairs.size(); i++) { + appendPair(buffer, slice[static_cast(pairs[i].key - lo)], pairs[i].other); + written++; + if (buffer.size() > 32 * 1024 * 1024) { + fwrite(buffer.data(), 1, buffer.size(), out); + buffer.clear(); + } + } + FileUtil::remove(Buckets::path(tmpB, b).c_str()); + } + if (buffer.empty() == false) fwrite(buffer.data(), 1, buffer.size(), out); + if (fclose(out) != 0) { + Debug(Debug::ERROR) << "Cannot close " << outTsv << "\n"; + EXIT(EXIT_FAILURE); + } + close(mapFd); + + Debug(Debug::INFO) << "Translated " << written << " assignments into " << outTsv << "\n"; + return EXIT_SUCCESS; +} diff --git a/src/test/TestKmerPartition.cpp b/src/test/TestKmerPartition.cpp index 2c9386ca6..87db7c8f3 100644 --- a/src/test/TestKmerPartition.cpp +++ b/src/test/TestKmerPartition.cpp @@ -321,6 +321,26 @@ static void testShuffleSizing() { check(waved.partitionCount <= large.partitionCount, "waves reduce the peak, so no more partitions are needed than without them"); + // A wave owns a contiguous slice of partitions, so the slices are only equal + // -- and peak scratch only really 1/W of the whole -- if W divides P. Both + // being powers of two is how that is guaranteed. + bool wavesDivide = true; + for (uint64_t budget = 4; budget <= 512; budget *= 2) { + // Sweep budgets from far below the k-mer volume to above it. Per-worker + // memory is large so nothing but the wave count can raise P, which is the + // case where the two used to disagree. + const KmerShuffleSizing s = + deriveKmerShuffleSizing(1000000000000ULL, 21, budget * TB, 0, 1024 * GB); + wavesDivide = wavesDivide && s.waveCount > 0 && + (s.waveCount & (s.waveCount - 1)) == 0 && + s.partitionCount >= s.waveCount && + s.partitionCount % s.waveCount == 0 && + // the largest slice really is within budget + (s.totalKmerBytes / s.partitionCount) * + (s.partitionCount / s.waveCount) <= budget * TB; + } + check(wavesDivide, "the wave count is a power of two that divides P at every budget"); + // P must always be a usable power of two. bool powerOfTwo = true; for (uint64_t seqs = 1000000; seqs <= 100000000000ULL; seqs *= 10) { diff --git a/src/test/TestLengthRankedPlan.cpp b/src/test/TestLengthRankedPlan.cpp index d2e0ac833..cb5e0e31f 100644 --- a/src/test/TestLengthRankedPlan.cpp +++ b/src/test/TestLengthRankedPlan.cpp @@ -63,11 +63,13 @@ static ChunkHistogram makeHistogram(uint64_t chunkIdx, uint64_t fileIdx, return histogram; } -static ChunkHistogram::Bucket bucket(uint64_t length, uint64_t count, uint64_t headerBytes) { +static ChunkHistogram::Bucket bucket(uint64_t length, uint64_t count, uint64_t headerBytes, + uint64_t accessionBytes = 0) { ChunkHistogram::Bucket b; b.length = length; b.count = count; b.headerBytes = headerBytes; + b.accessionBytes = accessionBytes; return b; } @@ -76,8 +78,8 @@ static ChunkHistogram::Bucket bucket(uint64_t length, uint64_t count, uint64_t h // one pins down that the arithmetic itself is the intended arithmetic. static void testWorkedExample() { std::vector histograms; - histograms.push_back(makeHistogram(0, 0, {bucket(5, 2, 20), bucket(10, 1, 12)})); - histograms.push_back(makeHistogram(1, 0, {bucket(5, 3, 33), bucket(7, 1, 9)})); + histograms.push_back(makeHistogram(0, 0, {bucket(5, 2, 20, 12), bucket(10, 1, 12, 7)})); + histograms.push_back(makeHistogram(1, 0, {bucket(5, 3, 33, 21), bucket(7, 1, 9, 5)})); std::vector plans; LengthRankedTotals totals = buildLengthRankedPlan(histograms, plans); @@ -86,6 +88,9 @@ static void testWorkedExample() { // 1*(10+2) + 1*(7+2) + 5*(5+2) = 12 + 9 + 35 check(totals.dataBytes == 56, "worked example totals 56 data bytes"); check(totals.headerBytes == 74, "worked example totals 74 header bytes"); + // Per line: the key digits, the accession, one digit of file index, two tabs + // and a newline. (7+1+4) + (5+1+4) + (12+2+8) + (21+3+12) = 12 + 10 + 22 + 36. + check(totals.lookupBytes == 80, "worked example totals 80 lookup bytes"); check(totals.maxSeqLen == 10, "worked example reports the longest sequence"); check(plans.size() == 2, "worked example produces one plan per chunk"); @@ -96,6 +101,8 @@ static void testWorkedExample() { const ChunkPlan::Entry &c0len10 = plans[0].entries[1]; check(c0len10.length == 10 && c0len10.keyStart == 0 && c0len10.dataOffset == 0 && c0len10.hdrOffset == 0, "longest sequence takes key 0 at offset 0"); + check(c0len10.lookupOffset == 0 && c0len10.lookupBytes == 12, + "longest sequence opens the lookup file"); const ChunkPlan::Entry &c1len7 = plans[1].entries[1]; check(c1len7.length == 7 && c1len7.keyStart == 1 && c1len7.dataOffset == 12 && c1len7.hdrOffset == 12, @@ -109,10 +116,12 @@ static void testWorkedExample() { "lower chunk index wins a length tie"); check(c1len5.keyStart == 4 && c1len5.dataOffset == 35 && c1len5.hdrOffset == 41, "higher chunk index follows within the same length"); + check(c0len5.lookupOffset == 22 && c1len5.lookupOffset == 44, + "lookup offsets follow the same key order"); } -// Materialises every (key, length, dataOffset, headerBytes) run the plan implies -// and checks the runs tile all three address spaces exactly. +// Materialises every (key, length, dataOffset, headerBytes, lookupOffset) run the +// plan implies and checks the runs tile all four address spaces exactly. static void testTilingInvariants() { srand(20260728); @@ -128,7 +137,8 @@ static void testTilingInvariants() { continue; } const uint64_t count = 1 + (rand() % 5); - buckets.push_back(bucket(length, count, count * (3 + (rand() % 7)))); + buckets.push_back(bucket(length, count, count * (3 + (rand() % 7)), + count * (2 + (rand() % 5)))); } histograms.push_back(makeHistogram(c, c % 3, buckets)); } @@ -140,38 +150,59 @@ static void testTilingInvariants() { // Flatten the plan into runs ordered by key. struct Run { uint64_t keyStart, count, length, dataOffset, hdrOffset, hdrBytes; + uint64_t lookupOffset, lookupBytes, accBytes, fileIdx; }; std::vector runs; for (size_t i = 0; i < plans.size(); i++) { for (size_t e = 0; e < plans[i].entries.size(); e++) { const ChunkPlan::Entry &entry = plans[i].entries[e]; uint64_t hdrBytes = 0; + uint64_t accBytes = 0; for (size_t b = 0; b < histograms[i].buckets.size(); b++) { if (histograms[i].buckets[b].length == entry.length) { hdrBytes = histograms[i].buckets[b].headerBytes; + accBytes = histograms[i].buckets[b].accessionBytes; } } Run run = {entry.keyStart, entry.count, entry.length, - entry.dataOffset, entry.hdrOffset, hdrBytes}; + entry.dataOffset, entry.hdrOffset, hdrBytes, + entry.lookupOffset, entry.lookupBytes, accBytes, plans[i].fileIdx}; runs.push_back(run); } } std::sort(runs.begin(), runs.end(), [](const Run &a, const Run &b) { return a.keyStart < b.keyStart; }); - uint64_t expectedKey = 0, expectedData = 0, expectedHdr = 0; + uint64_t expectedKey = 0, expectedData = 0, expectedHdr = 0, expectedLookup = 0; uint64_t previousLength = UINT64_MAX; bool keysTile = true, dataTiles = true, hdrTiles = true, lengthRanked = true; + bool lookupTiles = true, lookupWidths = true; for (size_t r = 0; r < runs.size(); r++) { keysTile = keysTile && runs[r].keyStart == expectedKey; dataTiles = dataTiles && runs[r].dataOffset == expectedData; hdrTiles = hdrTiles && runs[r].hdrOffset == expectedHdr; + lookupTiles = lookupTiles && runs[r].lookupOffset == expectedLookup; lengthRanked = lengthRanked && runs[r].length <= previousLength; + // The planner counts the key digits in closed form because the range + // is the whole database. Check it against actually rendering them, + // which is the part of the width that is not simply a sum. + char rendered[32]; + uint64_t brute = runs[r].accBytes; + for (uint64_t k = 0; k < runs[r].count; k++) { + brute += snprintf(rendered, sizeof(rendered), "%llu", + (unsigned long long)(runs[r].keyStart + k)); + } + brute += runs[r].count * + (snprintf(rendered, sizeof(rendered), "%llu", + (unsigned long long)runs[r].fileIdx) + 3); + lookupWidths = lookupWidths && brute == runs[r].lookupBytes; + previousLength = runs[r].length; expectedKey += runs[r].count; expectedData += runs[r].count * (runs[r].length + 2); expectedHdr += runs[r].hdrBytes; + expectedLookup += runs[r].lookupBytes; } if (trial == 0) { @@ -182,9 +213,13 @@ static void testTilingInvariants() { check(expectedKey == totals.seqCount, "totals agree with the tiled key count"); check(expectedData == totals.dataBytes, "totals agree with the tiled data size"); check(expectedHdr == totals.headerBytes, "totals agree with the tiled header size"); - } else if (!keysTile || !dataTiles || !hdrTiles || !lengthRanked || - expectedKey != totals.seqCount || expectedData != totals.dataBytes || - expectedHdr != totals.headerBytes) { + check(lookupTiles, "lookup byte ranges tile the lookup file exactly"); + check(lookupWidths, "reserved lookup width matches the rendered key digits"); + check(expectedLookup == totals.lookupBytes, "totals agree with the tiled lookup size"); + } else if (!keysTile || !dataTiles || !hdrTiles || !lengthRanked || !lookupTiles || + !lookupWidths || expectedKey != totals.seqCount || + expectedData != totals.dataBytes || expectedHdr != totals.headerBytes || + expectedLookup != totals.lookupBytes) { check(false, "tiling invariants hold on randomised trial " + std::to_string(trial)); return; } @@ -202,7 +237,9 @@ static void testTilingInvariants() { for (size_t e = 0; identical && e < plans[i].entries.size(); e++) { identical = shuffledPlans[i].entries[e].keyStart == plans[i].entries[e].keyStart && shuffledPlans[i].entries[e].dataOffset == plans[i].entries[e].dataOffset && - shuffledPlans[i].entries[e].hdrOffset == plans[i].entries[e].hdrOffset; + shuffledPlans[i].entries[e].hdrOffset == plans[i].entries[e].hdrOffset && + shuffledPlans[i].entries[e].lookupOffset == + plans[i].entries[e].lookupOffset; } } if (trial == 0) { @@ -216,7 +253,7 @@ static void testTilingInvariants() { } static void testRoundTrip(const std::string &dir) { - ChunkHistogram histogram = makeHistogram(7, 3, {bucket(4, 9, 40), bucket(11, 2, 18)}); + ChunkHistogram histogram = makeHistogram(7, 3, {bucket(4, 9, 40, 27), bucket(11, 2, 18, 9)}); histogram.nuclVotes = 5; histogram.sampleCount = 11; const std::string histPath = dir + "/chunk7.hist"; @@ -229,7 +266,8 @@ static void testRoundTrip(const std::string &dir) { for (size_t i = 0; same && i < loaded.buckets.size(); i++) { same = loaded.buckets[i].length == histogram.buckets[i].length && loaded.buckets[i].count == histogram.buckets[i].count && - loaded.buckets[i].headerBytes == histogram.buckets[i].headerBytes; + loaded.buckets[i].headerBytes == histogram.buckets[i].headerBytes && + loaded.buckets[i].accessionBytes == histogram.buckets[i].accessionBytes; } check(same, "chunk histogram survives a write/read round trip"); @@ -248,7 +286,9 @@ static void testRoundTrip(const std::string &dir) { loadedPlan.entries[i].count == plans[0].entries[i].count && loadedPlan.entries[i].keyStart == plans[0].entries[i].keyStart && loadedPlan.entries[i].dataOffset == plans[0].entries[i].dataOffset && - loadedPlan.entries[i].hdrOffset == plans[0].entries[i].hdrOffset; + loadedPlan.entries[i].hdrOffset == plans[0].entries[i].hdrOffset && + loadedPlan.entries[i].lookupOffset == plans[0].entries[i].lookupOffset && + loadedPlan.entries[i].lookupBytes == plans[0].entries[i].lookupBytes; } check(planSame, "chunk plan survives a write/read round trip"); } diff --git a/src/util/createdbparallel.cpp b/src/util/createdbparallel.cpp index 09ea86445..978733228 100644 --- a/src/util/createdbparallel.cpp +++ b/src/util/createdbparallel.cpp @@ -216,6 +216,7 @@ ChunkHistogram scanChunk(const std::string &filename, const Chunk &chunk, size_t std::vector lengthOf; std::vector countOf; std::vector headerBytesOf; + std::vector accessionBytesOf; KSeqBuffer kseq(buffer.data(), buffer.size()); std::string header; @@ -244,12 +245,16 @@ ChunkHistogram scanChunk(const std::string &filename, const Chunk &chunk, size_t lengthOf.insert(lengthOf.begin() + slot, length); countOf.insert(countOf.begin() + slot, 0); headerBytesOf.insert(headerBytesOf.begin() + slot, 0); + accessionBytesOf.insert(accessionBytesOf.begin() + slot, 0); } else { slot--; } countOf[slot]++; // DBWriter appends a NUL after the header text. headerBytesOf[slot] += header.length() + 1; + // The same extraction the emit pass will do, so the planner's byte count + // and the bytes actually written cannot disagree. + accessionBytesOf[slot] += Util::parseFastaHeader(header.c_str()).length(); if (sampleCount < testForNucSequence) { size_t nucleotideLike = 0; @@ -280,6 +285,7 @@ ChunkHistogram scanChunk(const std::string &filename, const Chunk &chunk, size_t histogram.buckets[i].length = lengthOf[i]; histogram.buckets[i].count = countOf[i]; histogram.buckets[i].headerBytes = headerBytesOf[i]; + histogram.buckets[i].accessionBytes = accessionBytesOf[i]; } return histogram; } @@ -289,6 +295,8 @@ ChunkHistogram scanChunk(const std::string &filename, const Chunk &chunk, size_t // a scatter of ~250 byte writes over a parallel filesystem. struct BucketOutput { uint64_t keyStart; + uint64_t lookupOffset; + std::string lookupText; uint64_t dataOffset; uint64_t hdrOffset; uint64_t written; @@ -300,7 +308,7 @@ struct BucketOutput { // Pass 2: rescan the chunk and write every sequence at its planned position. void emitChunk(const std::string &filename, const Chunk &chunk, const ChunkPlan &plan, - int seqFd, int hdrFd, int seqIdxFd, int hdrIdxFd) { + int seqFd, int hdrFd, int seqIdxFd, int hdrIdxFd, int lookupFd) { std::vector buffer; readChunk(filename, chunk, buffer); @@ -309,8 +317,10 @@ void emitChunk(const std::string &filename, const Chunk &chunk, const ChunkPlan buckets[i].keyStart = plan.entries[i].keyStart; buckets[i].dataOffset = plan.entries[i].dataOffset; buckets[i].hdrOffset = plan.entries[i].hdrOffset; + buckets[i].lookupOffset = plan.entries[i].lookupOffset; buckets[i].written = 0; } + const std::string fileIdxField = "\t" + SSTR(plan.fileIdx) + "\n"; KSeqBuffer kseq(buffer.data(), buffer.size()); std::string header; @@ -354,6 +364,13 @@ void emitChunk(const std::string &filename, const Chunk &chunk, const ChunkPlan bucket.hdrData.insert(bucket.hdrData.end(), header.begin(), header.end()); bucket.hdrData.push_back('\0'); + // The bucket's keys run from keyStart in the order it consumes them, so + // the key of this line is known without any shared counter. + bucket.lookupText.append(SSTR(bucket.keyStart + bucket.written)); + bucket.lookupText.push_back('\t'); + bucket.lookupText.append(Util::parseFastaHeader(header.c_str())); + bucket.lookupText.append(fileIdxField); + bucket.written++; } @@ -365,6 +382,13 @@ void emitChunk(const std::string &filename, const Chunk &chunk, const ChunkPlan << " but pass 1 counted " << plan.entries[i].count << "\n"; EXIT(EXIT_FAILURE); } + if (bucket.lookupText.size() != plan.entries[i].lookupBytes) { + Debug(Debug::ERROR) << "Chunk " << plan.chunkIdx << " built " + << bucket.lookupText.size() << " lookup bytes for length " + << plan.entries[i].length << " but the plan reserved " + << plan.entries[i].lookupBytes << "\n"; + EXIT(EXIT_FAILURE); + } if (bucket.written == 0) { continue; } @@ -374,6 +398,10 @@ void emitChunk(const std::string &filename, const Chunk &chunk, const ChunkPlan DenseIndex::entryOffset(bucket.keyStart), "sequence index"); writeAt(hdrIdxFd, bucket.hdrIndex.data(), bucket.hdrIndex.size() * sizeof(DenseIndex::Entry), DenseIndex::entryOffset(bucket.keyStart), "header index"); + if (lookupFd >= 0) { + writeAt(lookupFd, bucket.lookupText.data(), bucket.lookupText.size(), + bucket.lookupOffset, "lookup"); + } } } @@ -440,6 +468,7 @@ int createdbparallel(int argc, const char **argv, const Command &command) { } const std::string hdrDataFile = dataFile + "_h"; + const std::string lookupFile = dataFile + ".lookup"; // Derived, not passed, so every worker runs a byte-identical command line. const std::string coordDir = dataFile + ".coord"; if (FileUtil::directoryExists(coordDir.c_str()) == false) { @@ -497,6 +526,9 @@ int createdbparallel(int argc, const char **argv, const Command &command) { allocateFile(dataFile, totals.dataBytes); allocateFile(hdrDataFile, totals.headerBytes); + if (par.writeLookup) { + allocateFile(lookupFile, totals.lookupBytes); + } DenseIndex::createEmpty(dataFile, totals.seqCount, 0, totals.dataBytes, static_cast(totals.maxSeqLen + 2)); DenseIndex::createEmpty(hdrDataFile, totals.seqCount, 0, totals.headerBytes, 0); @@ -537,12 +569,16 @@ int createdbparallel(int argc, const char **argv, const Command &command) { const int hdrFd = openForWrite(hdrDataFile); const int seqIdxFd = openForWrite(DenseIndex::fileName(dataFile)); const int hdrIdxFd = openForWrite(DenseIndex::fileName(hdrDataFile)); + // Off by request only: at 1e12 sequences the lookup is ~30 TB, and the + // clustering path never reads it -- keys translate through the header + // database, which is addressed by the same dense keys. + const int lookupFd = par.writeLookup ? openForWrite(lookupFile) : -1; WorkQueue emitQueue(coordDir + "/emit.queue", static_cast(chunks.size())); runQueue(emitQueue, par.threads, workerId, [&](size_t chunkIdx) { const ChunkPlan plan = ChunkPlan::read(chunkPlanPath(coordDir, chunkIdx)); emitChunk(filenames[chunks[chunkIdx].fileIdx], chunks[chunkIdx], plan, - seqFd, hdrFd, seqIdxFd, hdrIdxFd); + seqFd, hdrFd, seqIdxFd, hdrIdxFd, lookupFd); }); // fsync before the sentinel, so a worker that finalises after a crash // cannot read a partially flushed database. @@ -550,10 +586,16 @@ int createdbparallel(int argc, const char **argv, const Command &command) { fsync(hdrFd); fsync(seqIdxFd); fsync(hdrIdxFd); + if (lookupFd >= 0) { + fsync(lookupFd); + } close(seqFd); close(hdrFd); close(seqIdxFd); close(hdrIdxFd); + if (lookupFd >= 0) { + close(lookupFd); + } } Debug(Debug::INFO) << "Emit pass done\n"; From 0842bb4987e7e264c2ebce301f5f198de295165a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Thu, 30 Jul 2026 12:57:27 +0000 Subject: [PATCH 07/27] Add clusterstsv conversion. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Björn Buschkämper --- data/workflow/linclustparallel.sh | 26 +- src/CommandDeclarations.h | 1 + src/MMseqsBase.cpp | 12 + src/commons/Parameters.cpp | 5 + src/commons/Parameters.h | 1 + src/linclust/CMakeLists.txt | 1 + src/linclust/alignparallel.cpp | 25 +- src/linclust/translatekeys.cpp | 386 ++++++++++++++++++++++++++++++ 8 files changed, 447 insertions(+), 10 deletions(-) create mode 100644 src/linclust/translatekeys.cpp diff --git a/data/workflow/linclustparallel.sh b/data/workflow/linclustparallel.sh index bc273dc29..094ad2a47 100755 --- a/data/workflow/linclustparallel.sh +++ b/data/workflow/linclustparallel.sh @@ -167,12 +167,32 @@ if notExists "$TMP/clu2.tsv"; then fi # ---- fold pass 2 into pass 1 ------------------------------------------------ -if notExists "$OUT"; then +# Keys, not accessions: every stage addresses sequences as dense keys, so this is +# the form the merge produces. It is translated below. +if notExists "$TMP/clu.keys.tsv"; then # shellcheck disable=SC2086 - "$MMSEQS" mergeclusterparallel "$DB" "$TMP/clu1.tsv" "$TMP/clu2.tsv" "$OUT.tmp" \ + "$MMSEQS" mergeclusterparallel "$DB" "$TMP/clu1.tsv" "$TMP/clu2.tsv" "$TMP/clu.keys.tsv.tmp" \ --threads $THREADS --split-memory-limit $SPLIT_MEMORY_LIMIT \ || fail "mergeclusterparallel died" - mv -f "$OUT.tmp" "$OUT" + mv -f "$TMP/clu.keys.tsv.tmp" "$TMP/clu.keys.tsv" +fi + +# ---- back to accession space ------------------------------------------------ +# Stock reaches this with createtsv, which needs a result database and a resident +# id->name table. Here it is a streaming join against the .lookup. Skipped when +# the database has none (--write-lookup 0), leaving the key-space result as the +# output rather than failing at the last step. +if notExists "$OUT"; then + if [ -f "$DB.lookup" ]; then + # shellcheck disable=SC2086 + "$MMSEQS" translatekeys "$TMP/clu.keys.tsv" "$DB.lookup" "$OUT.tmp" \ + --threads $THREADS --split-memory-limit $SPLIT_MEMORY_LIMIT \ + || fail "translatekeys died" + mv -f "$OUT.tmp" "$OUT" + else + echo "No $DB.lookup; leaving the result in database keys" + cp "$TMP/clu.keys.tsv" "$OUT" + fi fi echo "Wrote $OUT" diff --git a/src/CommandDeclarations.h b/src/CommandDeclarations.h index 15e6660be..3f19ae8f8 100644 --- a/src/CommandDeclarations.h +++ b/src/CommandDeclarations.h @@ -34,6 +34,7 @@ extern int greedycluster(int argc, const char **argv, const Command& command); extern int mergeclusterparallel(int argc, const char **argv, const Command& command); extern int createrepdb(int argc, const char **argv, const Command& command); extern int translatecluster(int argc, const char **argv, const Command& command); +extern int translatekeys(int argc, const char **argv, const Command& command); extern int makepaddedseqdb(int argc, const char **argv, const Command& command); extern int createindex(int argc, const char **argv, const Command& command); extern int createlinindex(int argc, const char **argv, const Command& command); diff --git a/src/MMseqsBase.cpp b/src/MMseqsBase.cpp index 3ccd70896..2b5ae4ec4 100644 --- a/src/MMseqsBase.cpp +++ b/src/MMseqsBase.cpp @@ -758,6 +758,18 @@ std::vector baseCommands = { CITATION_MMSEQS2, {{"clusterTsv", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::flatfile }, {"keyMap", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::flatfile }, {"clusterTsv", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::flatfile }}}, + {"translatekeys", translatekeys, &par.translatekeys, COMMAND_CLUSTER, + "Rewrite a clustering from database keys into accessions", + "# The distributed pipeline works in dense keys, so its clusters.tsv holds\n" + "# numbers. This joins it against the database's .lookup by streaming: each\n" + "# key column is translated in its own bucketed pass, and the lookup is read\n" + "# sequentially, so nothing per-key is ever resident.\n" + "mmseqs translatekeys clusters.tsv sequenceDB.lookup clusters_named.tsv\n", + "Martin Steinegger ", + " ", + CITATION_MMSEQS2, {{"clusterTsv", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::flatfile }, + {"lookupFile", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::flatfile }, + {"clusterTsv", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::flatfile }}}, {"kmersearch", kmersearch, &par.kmersearch, COMMAND_PREFILTER, "Find bottom-m-hashed k-mer matches between target and query DB", NULL, diff --git a/src/commons/Parameters.cpp b/src/commons/Parameters.cpp index a2e34dc39..375153a46 100644 --- a/src/commons/Parameters.cpp +++ b/src/commons/Parameters.cpp @@ -1006,6 +1006,11 @@ Parameters::Parameters(): translatecluster.push_back(&PARAM_THREADS); translatecluster.push_back(&PARAM_V); + // translatekeys + translatekeys.push_back(&PARAM_SPLIT_MEMORY_LIMIT); + translatekeys.push_back(&PARAM_THREADS); + translatekeys.push_back(&PARAM_V); + // makepaddedseqdb makepaddedseqdb.push_back(&PARAM_SUB_MAT); makepaddedseqdb.push_back(&PARAM_SCORE_BIAS); diff --git a/src/commons/Parameters.h b/src/commons/Parameters.h index d7f57fa75..82fa4c1ae 100644 --- a/src/commons/Parameters.h +++ b/src/commons/Parameters.h @@ -1248,6 +1248,7 @@ class Parameters { std::vector mergeclusterparallel; std::vector createrepdb; std::vector translatecluster; + std::vector translatekeys; std::vector makepaddedseqdb; std::vector convert2fasta; std::vector result2flat; diff --git a/src/linclust/CMakeLists.txt b/src/linclust/CMakeLists.txt index 941f3131a..17637eb31 100644 --- a/src/linclust/CMakeLists.txt +++ b/src/linclust/CMakeLists.txt @@ -7,6 +7,7 @@ set(linclust_source_files linclust/mergeclusterparallel.cpp linclust/createrepdb.cpp linclust/translatecluster.cpp + linclust/translatekeys.cpp linclust/CandidateEdge.cpp linclust/PartitionSequences.cpp linclust/KmerPartition.cpp diff --git a/src/linclust/alignparallel.cpp b/src/linclust/alignparallel.cpp index 809e95452..f7af50f17 100644 --- a/src/linclust/alignparallel.cpp +++ b/src/linclust/alignparallel.cpp @@ -222,12 +222,19 @@ struct FilterGate { // Runs stock's allpass loop for one candidate member. // -// The member's whole pass-1 cluster must align to the representative, each element -// on diagonal 0 -- ungapped first, then a full Smith-Waterman if that fails, exactly -// as Align2clust.cpp:669-726. A singleton pass-1 cluster passes trivially. +// The member's whole pass-1 cluster must align to the representative -- ungapped +// first, then a full Smith-Waterman if that fails, exactly as +// Align2clust.cpp:669-726. A singleton pass-1 cluster passes trivially. +// +// `elementDiagonal` is the caller's, because stock's two call sites disagree: the +// ungapped-accept gate seeds every element with the *candidate pair's* diagonal +// (`:680`), while the gapped-accept gate uses 0 (`:816`). Using the pair's +// diagonal for a different sequence reads like an oversight upstream, but parity +// is the goal, so both are reproduced as they are. Passing 0 in both places is +// what left 9 of 1,000,000 sequences under a different representative than stock. bool passesFilterGate(const FilterGate *gate, uint64_t subMember, Sequence &query, Sequence &element, BlockAligner &aligner, Matcher &matcher, - Parameters &par, unsigned int swMode) { + Parameters &par, unsigned int swMode, short elementDiagonal) { if (gate == NULL || subMember >= gate->keymap.size()) { return true; } @@ -249,7 +256,6 @@ bool passesFilterGate(const FilterGate *gate, uint64_t subMember, Sequence &quer if (Util::canBeCovered(par.covThr, par.covMode, query.L, element.L) == false) { return false; } - const short elementDiagonal = 0; BlockAligner::UngappedAln_res ua = aligner.ungappedAlign(&element, elementDiagonal); const bool hasEvalue = (ua.eval <= par.evalThr); const bool hasAlnLen = (ua.alnLen >= par.alnLenThr); @@ -503,8 +509,12 @@ int alignparallel(int argc, const char **argv, const Command &command) { seqId >= (par.seqIdThr - std::numeric_limits::epsilon()); if (hasAlnLen && hasCoverage && hasSeqId && hasEvalue) { + // Stock seeds this gate with the pair's diagonal + // (Align2clust.cpp:680), not with 0 as the gapped + // branch below does. if (passesFilterGate(gate, edges[e].getMember(), query, element, - aligner, matcher, par, swMode) == false) { + aligner, matcher, par, swMode, + 0) == false) { continue; } // The greedy ranks by alignment score, not k-mer count. @@ -562,8 +572,9 @@ int alignparallel(int argc, const char **argv, const Command &command) { // the reduce, so a representative is never its own member. if (Alignment::checkCriteria(result, false, par.evalThr, par.seqIdThr, par.alnLenThr, par.covMode, par.covThr)) { + // 0 here, matching Align2clust.cpp:816. if (passesFilterGate(gate, edges[e].getMember(), query, element, - aligner, matcher, par, swMode) == false) { + aligner, matcher, par, swMode, 0) == false) { continue; } edges[e].score = diff --git a/src/linclust/translatekeys.cpp b/src/linclust/translatekeys.cpp new file mode 100644 index 000000000..affbe7975 --- /dev/null +++ b/src/linclust/translatekeys.cpp @@ -0,0 +1,386 @@ +/* + * translatekeys -- rewrites a clustering from database keys into accessions. + * + * The pipeline works in dense keys throughout, so its final `clusters.tsv` reads + * `027`. Stock reaches accessions with `createtsv`, which needs a result + * *database* and builds a resident id->name table; both are exactly the per-key + * state that cannot exist at 1e11 sequences. This does the same join streaming. + * + * The map is `.lookup`, which `createdbparallel` writes in ascending key + * order. Unlike the fixed-width key map `translatecluster` reads, its lines vary + * in width, so a key's accession is at no computable offset and cannot be fetched + * by pread. What replaces that is ordering: buckets are visited in ascending key + * order and tile the key space, so one *sequential* pass over the lookup serves a + * whole column -- the cursor only ever moves forward. + * + * A clustering has two key columns and only one can be in order at a time, so + * each gets its own bucketed pass, as in translatecluster: + * + * 1. bucket the input by member key (fixed-width spill) + * 2. stream the lookup, translate the member, re-bucket + * by representative key (variable-width spill) + * 3. stream the lookup again, translate the representative, emit + * + * Two sequential reads of the lookup, two of the input, and peak memory of one + * bucket -- never the whole map. + */ +#include "Command.h" +#include "Debug.h" +#include "FileUtil.h" +#include "Parameters.h" +#include "Util.h" + +#include +#include +#include +#include +#include + +namespace { + +// Spill file holding (key, payload) records whose payload is a key too. +struct __attribute__((__packed__)) KeyPair { + uint64_t key; // the column being translated in this pass + uint64_t other; // the column carried through untouched +}; + +// Spill file holding (key, text) records. Needed for the second pass, where the +// column carried through has already become an accession and so varies in width. +class TextBuckets { +public: + TextBuckets(const std::string &prefix, unsigned int count, size_t flushBytes = 8 << 20) + : prefix(prefix), count(count), flushBytes(flushBytes), closed(false) { + files.assign(count, NULL); + buffers.resize(count); + } + ~TextBuckets() { close(); } + + void append(unsigned int b, uint64_t key, const char *text, size_t length) { + std::string &buf = buffers[b]; + const uint32_t len = static_cast(length); + buf.append(reinterpret_cast(&key), sizeof(key)); + buf.append(reinterpret_cast(&len), sizeof(len)); + buf.append(text, length); + if (buf.size() >= flushBytes) { + flush(b); + } + } + + void close() { + if (closed) return; + closed = true; + for (unsigned int b = 0; b < count; b++) { + flush(b); + if (files[b] != NULL) { fclose(files[b]); files[b] = NULL; } + } + } + + static std::string path(const std::string &prefix, unsigned int b) { + return prefix + "." + SSTR(b); + } + + // Reads one bucket back as (key, accession) pairs. + static void read(const std::string &prefix, unsigned int b, + std::vector > &out) { + out.clear(); + const std::string p = path(prefix, b); + if (FileUtil::fileExists(p.c_str()) == false) return; + const size_t bytes = FileUtil::getFileSize(p); + if (bytes == 0) return; + std::string blob(bytes, '\0'); + FILE *f = FileUtil::openFileOrDie(p.c_str(), "rb", true); + if (fread(&blob[0], 1, bytes, f) != bytes) { + Debug(Debug::ERROR) << "Cannot read " << p << "\n"; + EXIT(EXIT_FAILURE); + } + fclose(f); + size_t at = 0; + while (at + sizeof(uint64_t) + sizeof(uint32_t) <= bytes) { + uint64_t key; + uint32_t len; + memcpy(&key, blob.data() + at, sizeof(key)); + at += sizeof(key); + memcpy(&len, blob.data() + at, sizeof(len)); + at += sizeof(len); + out.push_back(std::make_pair(key, blob.substr(at, len))); + at += len; + } + } + +private: + void flush(unsigned int b) { + if (buffers[b].empty()) return; + if (files[b] == NULL) { + files[b] = fopen(path(prefix, b).c_str(), "wb"); + if (files[b] == NULL) { + Debug(Debug::ERROR) << "Cannot open " << path(prefix, b) << ": " << strerror(errno) + << "\n"; + EXIT(EXIT_FAILURE); + } + } + if (fwrite(buffers[b].data(), 1, buffers[b].size(), files[b]) != buffers[b].size()) { + Debug(Debug::ERROR) << "Cannot write bucket " << b << "\n"; + EXIT(EXIT_FAILURE); + } + buffers[b].clear(); + } + + std::string prefix; + unsigned int count; + size_t flushBytes; + std::vector buffers; + std::vector files; + bool closed; +}; + +// Fixed-width spill, for the first pass where both columns are still keys. +class KeyBuckets { +public: + KeyBuckets(const std::string &prefix, unsigned int count, size_t bufferPairs = 1 << 16) + : prefix(prefix), count(count), bufferPairs(bufferPairs), closed(false) { + files.assign(count, NULL); + buffers.resize(count); + } + ~KeyBuckets() { close(); } + + void append(unsigned int b, uint64_t key, uint64_t other) { + KeyPair p; + p.key = key; + p.other = other; + buffers[b].push_back(p); + if (buffers[b].size() >= bufferPairs) flush(b); + } + + void close() { + if (closed) return; + closed = true; + for (unsigned int b = 0; b < count; b++) { + flush(b); + if (files[b] != NULL) { fclose(files[b]); files[b] = NULL; } + } + } + + static std::string path(const std::string &prefix, unsigned int b) { + return prefix + "." + SSTR(b); + } + + static std::vector read(const std::string &prefix, unsigned int b) { + std::vector out; + const std::string p = path(prefix, b); + if (FileUtil::fileExists(p.c_str()) == false) return out; + const size_t bytes = FileUtil::getFileSize(p); + if (bytes == 0 || bytes % sizeof(KeyPair) != 0) return out; + out.resize(bytes / sizeof(KeyPair)); + FILE *f = FileUtil::openFileOrDie(p.c_str(), "rb", true); + if (fread(out.data(), sizeof(KeyPair), out.size(), f) != out.size()) { + Debug(Debug::ERROR) << "Cannot read " << p << "\n"; + EXIT(EXIT_FAILURE); + } + fclose(f); + return out; + } + +private: + void flush(unsigned int b) { + if (buffers[b].empty()) return; + if (files[b] == NULL) { + files[b] = fopen(path(prefix, b).c_str(), "wb"); + if (files[b] == NULL) { + Debug(Debug::ERROR) << "Cannot open " << path(prefix, b) << ": " << strerror(errno) + << "\n"; + EXIT(EXIT_FAILURE); + } + } + if (fwrite(buffers[b].data(), sizeof(KeyPair), buffers[b].size(), files[b]) != + buffers[b].size()) { + Debug(Debug::ERROR) << "Cannot write bucket " << b << "\n"; + EXIT(EXIT_FAILURE); + } + buffers[b].clear(); + } + + std::string prefix; + unsigned int count; + size_t bufferPairs; + std::vector > buffers; + std::vector files; + bool closed; +}; + +// Forward-only cursor over a `.lookup`. +// +// Keys ascend, and every caller asks for a higher range than the last, so the +// file is read once from start to end however many buckets there are. Gaps are +// tolerated (a key map from a sparse database leaves holes); a hole is only an +// error if a clustering actually refers to that key, which slice()'s caller +// detects by finding an empty accession. +class LookupCursor { +public: + explicit LookupCursor(const std::string &path) : line(NULL), cap(0), pending(false), pendingKey(0) { + file = FileUtil::openFileOrDie(path.c_str(), "r", true); + } + ~LookupCursor() { + free(line); + if (file != NULL) fclose(file); + } + + // Fills out[0 .. hi-lo) with the accessions of keys lo .. hi-1. + void slice(uint64_t lo, uint64_t hi, std::vector &out) { + out.clear(); + out.resize(static_cast(hi - lo)); + while (true) { + if (pending == false) { + if (getline(&line, &cap, file) <= 0) return; + char *tab = strchr(line, '\t'); + if (tab == NULL) continue; + pendingKey = strtoull(line, NULL, 10); + char *end = strchr(tab + 1, '\t'); + pendingAccession.assign(tab + 1, end != NULL ? (size_t)(end - tab - 1) + : strlen(tab + 1)); + pending = true; + } + if (pendingKey >= hi) return; // belongs to a later bucket, keep it + if (pendingKey >= lo) { + out[static_cast(pendingKey - lo)].swap(pendingAccession); + } + pending = false; // below lo: already consumed by an earlier bucket + } + } + +private: + FILE *file; + char *line; + size_t cap; + bool pending; + uint64_t pendingKey; + std::string pendingAccession; +}; + +const std::string &accessionOf(const std::vector &slice, uint64_t key, uint64_t lo, + const std::string &lookupFile) { + const std::string &name = slice[static_cast(key - lo)]; + if (name.empty()) { + Debug(Debug::ERROR) << "Key " << key << " of the clustering has no entry in " << lookupFile + << ". The clustering and the lookup are from different databases.\n"; + EXIT(EXIT_FAILURE); + } + return name; +} + +} // namespace + +int translatekeys(int argc, const char **argv, const Command &command) { + Parameters &par = Parameters::getInstance(); + par.parseParameters(argc, argv, command, true, 0, 0); + + const std::string inTsv = par.db1; + const std::string lookupFile = par.db2; + const std::string outTsv = par.db3; + + // Sized so one slice of accessions fits the memory budget. An accession is + // ~30 bytes plus the std::string that holds it, so 64 bytes per key is a safe + // working figure; being wrong only changes how many buckets are used. + uint64_t keyCount = 0; + { + LookupCursor probe(lookupFile); + // Counting lines is one extra sequential pass, but the alternative is + // guessing the key space from the clustering, which need not mention the + // highest key at all. + FILE *f = FileUtil::openFileOrDie(lookupFile.c_str(), "r", true); + char *line = NULL; + size_t cap = 0; + while (getline(&line, &cap, f) > 0) keyCount++; + free(line); + fclose(f); + } + if (keyCount == 0) { + Debug(Debug::ERROR) << lookupFile << " is empty\n"; + EXIT(EXIT_FAILURE); + } + const uint64_t targetBytes = + std::max(Util::computeMemory(par.splitMemoryLimit) / 8, 1ULL * 1024 * 1024); + unsigned int buckets = 1; + while (buckets < 65536 && (keyCount / buckets) * 64 > targetBytes) { + buckets *= 2; + } + const uint64_t span = (keyCount + buckets - 1) / buckets; + Debug(Debug::INFO) << "Translating " << keyCount << " keys over " << buckets << " buckets of " + << span << "\n"; + + const std::string tmpA = outTsv + ".bymember"; + const std::string tmpB = outTsv + ".byrep"; + + // Pass 1: bucket by member key. + { + KeyBuckets byMember(tmpA, buckets); + FILE *f = FileUtil::openFileOrDie(inTsv.c_str(), "r", true); + char *line = NULL; + size_t cap = 0; + while (getline(&line, &cap, f) > 0) { + char *tab = strchr(line, '\t'); + if (tab == NULL) continue; + const uint64_t rep = strtoull(line, NULL, 10); + const uint64_t member = strtoull(tab + 1, NULL, 10); + byMember.append(static_cast(member / span), member, rep); + } + free(line); + fclose(f); + } + + // Pass 2: translate the member, re-bucket on the representative. + { + LookupCursor cursor(lookupFile); + TextBuckets byRep(tmpB, buckets); + std::vector slice; + for (unsigned int b = 0; b < buckets; b++) { + const uint64_t lo = b * span; + if (lo >= keyCount) break; + const uint64_t hi = std::min(lo + span, keyCount); + cursor.slice(lo, hi, slice); + const std::vector pairs = KeyBuckets::read(tmpA, b); + for (size_t i = 0; i < pairs.size(); i++) { + const std::string &name = accessionOf(slice, pairs[i].key, lo, lookupFile); + byRep.append(static_cast(pairs[i].other / span), pairs[i].other, + name.c_str(), name.size()); + } + FileUtil::remove(KeyBuckets::path(tmpA, b).c_str()); + } + } + + // Pass 3: translate the representative and emit. + LookupCursor cursor(lookupFile); + FILE *out = FileUtil::openAndDelete(outTsv.c_str(), "w"); + std::string buffer; + buffer.reserve(64 * 1024 * 1024); + std::vector slice; + std::vector > entries; + uint64_t written = 0; + for (unsigned int b = 0; b < buckets; b++) { + const uint64_t lo = b * span; + if (lo >= keyCount) break; + const uint64_t hi = std::min(lo + span, keyCount); + cursor.slice(lo, hi, slice); + TextBuckets::read(tmpB, b, entries); + for (size_t i = 0; i < entries.size(); i++) { + const std::string &repName = accessionOf(slice, entries[i].first, lo, lookupFile); + buffer.append(repName); + buffer.push_back('\t'); + buffer.append(entries[i].second); + buffer.push_back('\n'); + written++; + if (buffer.size() > 32 * 1024 * 1024) { + fwrite(buffer.data(), 1, buffer.size(), out); + buffer.clear(); + } + } + FileUtil::remove(TextBuckets::path(tmpB, b).c_str()); + } + if (buffer.empty() == false) fwrite(buffer.data(), 1, buffer.size(), out); + if (fclose(out) != 0) { + Debug(Debug::ERROR) << "Cannot close " << outTsv << "\n"; + EXIT(EXIT_FAILURE); + } + + Debug(Debug::INFO) << "Translated " << written << " assignments into " << outTsv << "\n"; + return EXIT_SUCCESS; +} From 19621526f438ee6c89cd8322833a80d1cb3775c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Thu, 30 Jul 2026 13:56:01 +0000 Subject: [PATCH 08/27] Seed the pass-2 filter gate with the pair's diagonal. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Björn Buschkämper --- src/linclust/alignparallel.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/linclust/alignparallel.cpp b/src/linclust/alignparallel.cpp index f7af50f17..a689bddb0 100644 --- a/src/linclust/alignparallel.cpp +++ b/src/linclust/alignparallel.cpp @@ -514,7 +514,7 @@ int alignparallel(int argc, const char **argv, const Command &command) { // branch below does. if (passesFilterGate(gate, edges[e].getMember(), query, element, aligner, matcher, par, swMode, - 0) == false) { + edges[e].diagonal) == false) { continue; } // The greedy ranks by alignment score, not k-mer count. From 9cbe59c4adb44e6192ae2696f4cd0d7a4db36fe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Fri, 31 Jul 2026 09:22:44 +0000 Subject: [PATCH 09/27] Various smaller bugfixes. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Björn Buschkämper --- data/workflow/linclustparallel.sh | 32 +++- src/commons/LengthRankedPlan.cpp | 8 +- src/commons/ParallelCoordination.cpp | 20 +++ src/commons/ParallelCoordination.h | 39 ++++- src/commons/Parameters.cpp | 18 ++- src/commons/Parameters.h | 4 +- src/linclust/CandidateEdge.cpp | 39 ++++- src/linclust/CandidateEdge.h | 19 ++- src/linclust/KmerPartition.cpp | 56 ++++++- src/linclust/alignparallel.cpp | 6 +- src/linclust/createrepdb.cpp | 58 ++++++- src/linclust/greedycluster.cpp | 18 ++- src/linclust/kmermatcherparallel.cpp | 54 ++++++- src/linclust/kmerreduceparallel.cpp | 79 ++++++++-- src/linclust/mergeclusterparallel.cpp | 18 ++- src/linclust/translatecluster.cpp | 28 +++- src/linclust/translatekeys.cpp | 208 ++++++++++++++++++++------ src/test/TestKmerPartition.cpp | 13 +- 18 files changed, 624 insertions(+), 93 deletions(-) diff --git a/data/workflow/linclustparallel.sh b/data/workflow/linclustparallel.sh index 094ad2a47..e3ffcab74 100755 --- a/data/workflow/linclustparallel.sh +++ b/data/workflow/linclustparallel.sh @@ -46,6 +46,25 @@ notExists() { [ ! -f "$1" ]; } fail() { echo "Error: $1"; exit 1; } +# Bytes currently sitting on the scratch filesystem for this run. +# +# The wave count has to be derived against what is actually free, not against the +# whole budget: pass 2 runs with all of pass 1's surviving output still on disk, +# and sizing it as though the filesystem were empty is what made a 1e11 run derive +# a single wave and then peak at ~1.9x its ceiling. Measuring beats modelling here +# -- it needs no per-stage accounting and stays right when the pipeline changes. +scratchUsed() { + du -sb "$TMP" "$DB" 2>/dev/null | awk '{s += $1} END {print s + 0 "B"}' +} + +# Deletes an intermediate whose consumers have all finished. Nothing downstream +# reopens these, and they are the bulk of peak scratch: at 100M the candidate +# edges alone are 21 GB and the pass-1 alignments 3 GB. +dropIntermediate() { + [ -n "$KEEP_INTERMEDIATE" ] && return 0 + rm -rf "$@" +} + # Extraction waves. A wave re-extracts every k-mer but keeps only its own slice of # partition space, so peak scratch is the whole shuffle divided by the wave count, # paid for with that many passes over the sequences. How many are needed follows @@ -64,7 +83,10 @@ mapReduceWaves() { # $1 sequence DB, $2 k-mer dir, $3 edge dir, $4... extra map arguments _db="$1"; _kmer="$2"; _edges="$3"; shift 3 # shellcheck disable=SC2086 + _used=$(scratchUsed) + # shellcheck disable=SC2086 $RUNNER "$MMSEQS" kmermatcherparallel "$_db" "$_kmer" $KMER_COMMON "$@" --kmer-wave 0 \ + --scratch-used "$_used" \ || fail "kmermatcherparallel died" # shellcheck disable=SC2086 $RUNNER "$MMSEQS" kmerreduceparallel "$_db" "$_kmer" "$_edges" $REDUCE_PAR --kmer-wave 0 \ @@ -76,6 +98,7 @@ mapReduceWaves() { echo "--- extraction wave $_w of $_waves ---" # shellcheck disable=SC2086 $RUNNER "$MMSEQS" kmermatcherparallel "$_db" "$_kmer" $KMER_COMMON "$@" --kmer-wave $_w \ + --scratch-used "$_used" \ || fail "kmermatcherparallel (wave $_w) died" # shellcheck disable=SC2086 $RUNNER "$MMSEQS" kmerreduceparallel "$_db" "$_kmer" "$_edges" $REDUCE_PAR --kmer-wave $_w \ @@ -106,7 +129,10 @@ ALIGN_PAR="--min-seq-id $MIN_SEQ_ID --min-aln-len 0 --seq-id-mode 0 -e $EVAL -c DB="$INPUT" if notExists "$INPUT.dbtype"; then DB="$TMP/db" - if notExists "$DB.dbtype"; then + # Guarded on the finalize sentinel, not on .dbtype: createdbparallel writes + # .dbtype before the text indices, so a death in that window leaves a database + # that looks complete and is not. The sentinel is written last. + if notExists "$DB.coord/finalize.done"; then # shellcheck disable=SC2086 $RUNNER "$MMSEQS" createdbparallel "$INPUT" "$DB" --threads $THREADS \ || fail "createdbparallel died" @@ -129,6 +155,9 @@ if notExists "$TMP/clu1.tsv"; then || fail "greedycluster (pass 1) died" mv -f "$TMP/clu1.tsv.tmp" "$TMP/clu1.tsv" fi +# Both are dead once clu1.tsv exists, and together they are the largest thing pass +# 1 leaves behind for pass 2 to be sized around. +dropIntermediate "$TMP/edges1" "$TMP/aln1" # ---- pass 2, over the representatives -------------------------------------- # Re-keyed densely rather than sub-set, because every stage addresses keys as @@ -157,6 +186,7 @@ if notExists "$TMP/clu2_sub.tsv"; then || fail "greedycluster (pass 2) died" mv -f "$TMP/clu2_sub.tsv.tmp" "$TMP/clu2_sub.tsv" fi +dropIntermediate "$TMP/edges2" "$TMP/aln2" "$TMP/kmer2" if notExists "$TMP/clu2.tsv"; then # shellcheck disable=SC2086 diff --git a/src/commons/LengthRankedPlan.cpp b/src/commons/LengthRankedPlan.cpp index a38fac7d8..ecaa99a17 100644 --- a/src/commons/LengthRankedPlan.cpp +++ b/src/commons/LengthRankedPlan.cpp @@ -3,6 +3,8 @@ #include "Debug.h" #include "FileUtil.h" +#include + #include #include @@ -28,7 +30,11 @@ void writeBlock(const std::string &path, const FileHeader &header, const void *entries, size_t entryBytes) { // Write to a temporary and rename, so a worker that dies mid-write leaves no // truncated file that a later reader would mistake for a complete one. - std::string tmp = path + ".tmp"; + // Tagged with the pid: two workers redoing the same chunk would otherwise + // both open the same .tmp, and one could truncate the other's in-flight write + // before renaming the hole-ridden result into place. The rename itself is + // atomic, so a private temp makes the whole publish atomic. + std::string tmp = path + ".tmp." + SSTR(getpid()); FILE *file = FileUtil::openAndDelete(tmp.c_str(), "wb"); if (fwrite(&header, sizeof(FileHeader), 1, file) != 1) { Debug(Debug::ERROR) << "Cannot write header to " << tmp << "\n"; diff --git a/src/commons/ParallelCoordination.cpp b/src/commons/ParallelCoordination.cpp index 619d620dd..ef5aface6 100644 --- a/src/commons/ParallelCoordination.cpp +++ b/src/commons/ParallelCoordination.cpp @@ -366,6 +366,26 @@ int64_t WorkQueue::getDoneCount() { return static_cast(header.doneCount); } +// True while some item is held under a lease that has not yet expired. +// +// drain() uses this to tell "nobody has finished anything for a long time +// because the run is stuck" from "because one item legitimately takes hours". +// Heartbeats keep a live holder's expiry in the future, so a lease that is still +// valid means a worker is still on it. +bool WorkQueue::hasLiveClaim() { + const int64_t now = nowSeconds(); + lock.lock(); + bool live = false; + for (int64_t i = 0; i < itemCount && live == false; i++) { + const Record record = readRecordLocked(i); + if (record.state == CLAIMED && static_cast(record.leaseExpiry) > now) { + live = true; + } + } + lock.unlock(); + return live; +} + bool WorkQueue::allDone() { return getDoneCount() >= itemCount; } diff --git a/src/commons/ParallelCoordination.h b/src/commons/ParallelCoordination.h index d747a94eb..f5f930676 100644 --- a/src/commons/ParallelCoordination.h +++ b/src/commons/ParallelCoordination.h @@ -5,7 +5,9 @@ #include #include #include +#include #include +#include // Shared-filesystem coordination for multi-node MMseqs2 stages. // @@ -126,6 +128,9 @@ class WorkQueue { int64_t getDoneCount(); bool allDone(); + // True while some item is held under an unexpired lease. + bool hasLiveClaim(); + // Polls until every item is DONE. Returns false if no progress was made and // no item was claimable for stallSeconds, which means the remaining work is // held by workers that are gone but whose leases have not yet expired, or @@ -157,7 +162,30 @@ class WorkQueue { while (true) { const int64_t item = claim(workerId); if (item >= 0) { + // Heartbeat for the duration of the item. Without it any item + // taking longer than the lease -- which at 1e11 is every reduce + // partition and every align bucket, both hours of work -- looks + // abandoned, gets re-claimed, and is then run by two workers at + // once. The stage's output is not written per worker, so that is + // silent corruption rather than merely wasted effort. + std::atomic running(true); + std::thread heartbeat([this, item, workerId, &running]() { + const int64_t every = DEFAULT_LEASE_SECONDS / 3; + int64_t slept = 0; + while (running.load()) { + sleep(1); + if (++slept < every) { + continue; + } + slept = 0; + if (running.load()) { + renew(item, workerId); + } + } + }); body(static_cast(item)); + running.store(false); + heartbeat.join(); complete(item, workerId); continue; } @@ -168,6 +196,10 @@ class WorkQueue { if (done != lastDone) { lastDone = done; lastProgress = static_cast(time(NULL)); + } else if (hasLiveClaim()) { + // Someone is still working, and heartbeating to say so. A stage + // whose last item takes hours must not be declared stalled. + lastProgress = static_cast(time(NULL)); } else if (stallSeconds > 0 && static_cast(time(NULL)) - lastProgress > static_cast(stallSeconds)) { @@ -177,7 +209,12 @@ class WorkQueue { } } - static const int64_t DEFAULT_LEASE_SECONDS = 1800; + // Bounded by how long a dead worker's item should stay unclaimable, not by + // how long an item takes: drain() heartbeats every DEFAULT_LEASE_SECONDS/3 + // for as long as the item runs, so a live holder never lets its lease lapse. + // At 1800 s a single killed worker idled an entire measured run for 30 + // minutes before anyone could redo its item. + static const int64_t DEFAULT_LEASE_SECONDS = 300; private: // On-disk layout. Both structs are written and read verbatim; sizes are diff --git a/src/commons/Parameters.cpp b/src/commons/Parameters.cpp index 375153a46..28286c27c 100644 --- a/src/commons/Parameters.cpp +++ b/src/commons/Parameters.cpp @@ -1,4 +1,6 @@ #include "Parameters.h" + +#include #include "Util.h" #include "DistanceCalculator.h" #include "Debug.h" @@ -59,6 +61,7 @@ Parameters::Parameters(): PARAM_KMER_WAVE(PARAM_KMER_WAVE_ID, "--kmer-wave", "K-mer wave", "Which extraction wave to write, when the scratch budget needs more than one. Each wave re-extracts every k-mer but keeps only its own slice of the partition space, so peak scratch is the whole shuffle divided by the wave count. Default -1 requires a single wave. Run waves 0..W-1, reducing each before starting the next.", typeid(int), (void *) &kmerWave, "^-?[0-9]+$", MMseqsParameter::COMMAND_EXPERT), PARAM_KEY_MAP(PARAM_KEY_MAP_ID, "--key-map", "Sub-key map", "Maps this database's dense sub-keys back to the original keys, as written by createrepdb. Needed with --filter-cludb-file when the pass runs on a re-keyed representative database.", typeid(std::string), (void *) &keyMapFile, "", MMseqsParameter::COMMAND_ALIGN | MMseqsParameter::COMMAND_EXPERT), PARAM_SCRATCH_BUDGET(PARAM_SCRATCH_BUDGET_ID, "--scratch-budget", "Scratch budget", "Total scratch the run may occupy. The k-mer extraction wave count and the partition count are derived from this together with --split-memory-limit, rather than set by hand. Default (0) for a single wave. E.g. 100T, 500T", typeid(ByteParser), (void *) &scratchBudget, "^(0|[1-9]{1}[0-9]*(B|K|M|G|T)?)$", MMseqsParameter::COMMAND_EXPERT), + PARAM_SCRATCH_USED(PARAM_SCRATCH_USED_ID, "--scratch-used", "Scratch already used", "Bytes of --scratch-budget already occupied when this stage starts. The workflow measures it, so a later pass accounts for what earlier passes left on disk. 0 derives it from the input database alone.", typeid(ByteParser), (void *) &scratchUsed, "^([1-9]{1}[0-9]*(B|K|M|G|T)?)|0$", MMseqsParameter::COMMAND_EXPERT), PARAM_DISK_SPACE_LIMIT(PARAM_DISK_SPACE_LIMIT_ID, "--disk-space-limit", "Disk space limit", "Set max disk space to use for reverse profile searches. E.g. 800B, 5K, 10M, 1G. Default (0) to all available disk space in the temp folder", typeid(ByteParser), (void *) &diskSpaceLimit, "^(0|[1-9]{1}[0-9]*(B|K|M|G|T)?)$", MMseqsParameter::COMMAND_COMMON | MMseqsParameter::COMMAND_PREFILTER | MMseqsParameter::COMMAND_EXPERT), PARAM_SPLIT_AMINOACID(PARAM_SPLIT_AMINOACID_ID, "--split-aa", "Split by amino acid", "Try to find the best split boundaries by entry lengths", typeid(bool), (void *) &splitAA, "$", MMseqsParameter::COMMAND_EXPERT), PARAM_SUB_MAT(PARAM_SUB_MAT_ID, "--sub-mat", "Substitution matrix", "Substitution matrix file", typeid(MultiParam>), (void *) &scoringMatrixFile, "", MMseqsParameter::COMMAND_COMMON | MMseqsParameter::COMMAND_EXPERT), @@ -943,6 +946,7 @@ Parameters::Parameters(): kmermatcherparallel.push_back(&PARAM_IGNORE_MULTI_KMER); kmermatcherparallel.push_back(&PARAM_SPLIT_MEMORY_LIMIT); kmermatcherparallel.push_back(&PARAM_SCRATCH_BUDGET); + kmermatcherparallel.push_back(&PARAM_SCRATCH_USED); kmermatcherparallel.push_back(&PARAM_KMER_WAVE); kmermatcherparallel.push_back(&PARAM_THREADS); kmermatcherparallel.push_back(&PARAM_COMPRESSED); @@ -2485,9 +2489,18 @@ void Parameters::checkIfDatabaseIsValid(const Command& command, int argc, const } else if (db.accessMode == db.ACCESS_MODE_OUTPUT) { if (db.validator == &DbValidator::directory) { if (FileUtil::directoryExists(filenames[fileIdx].c_str()) == false) { - if (FileUtil::makeDir(filenames[fileIdx].c_str()) == false) { + // Racing workers may both reach this. makeDir is a bare mkdir, + // so the loser gets EEXIST and would exit -- which, with a + // Slurm array starting every worker at once, is nearly every + // launch of the shared-filesystem stages. Only a failure that + // also leaves no directory behind is real. Same rule the + // linclust modules already apply (KmerPartition.cpp, + // CandidateEdge.cpp). + if (FileUtil::makeDir(filenames[fileIdx].c_str()) == false && + FileUtil::directoryExists(filenames[fileIdx].c_str()) == false) { printParameters(command.cmd, argc, argv, *command.params); - Debug(Debug::ERROR) << "Cannot create temporary directory " << filenames[fileIdx] << "\n"; + Debug(Debug::ERROR) << "Cannot create temporary directory " << filenames[fileIdx] + << ": " << strerror(errno) << "\n"; EXIT(EXIT_FAILURE); } else { Debug(Debug::INFO) << "Create directory " << filenames[fileIdx] << "\n"; @@ -2605,6 +2618,7 @@ void Parameters::setDefaults() { kmerWave = -1; keyMapFile = ""; scratchBudget = 0; + scratchUsed = 0; diskSpaceLimit = 0; splitAA = false; spacedKmerPattern = ""; diff --git a/src/commons/Parameters.h b/src/commons/Parameters.h index 82fa4c1ae..c514f982b 100644 --- a/src/commons/Parameters.h +++ b/src/commons/Parameters.h @@ -442,7 +442,8 @@ class Parameters { size_t chunkSize; // Input bytes one createdbparallel work item covers std::string keyMapFile; // alignparallel: sub-key -> original key for the filter gate int kmerWave; // kmermatcherparallel: which extraction wave to write - size_t scratchBudget; // Scratch ceiling the k-mer wave count is derived from + size_t scratchBudget; + size_t scratchUsed; // Bytes of the budget already occupied when a stage starts size_t diskSpaceLimit; // Maximum disk space in bytes for sliced reverse profile search bool splitAA; // Split database by amino acid count instead int preloadMode; // Preload mode of database @@ -835,6 +836,7 @@ class Parameters { PARAMETER(PARAM_KMER_WAVE) PARAMETER(PARAM_KEY_MAP) PARAMETER(PARAM_SCRATCH_BUDGET) + PARAMETER(PARAM_SCRATCH_USED) PARAMETER(PARAM_DISK_SPACE_LIMIT) PARAMETER(PARAM_SPLIT_AMINOACID) PARAMETER(PARAM_SUB_MAT) diff --git a/src/linclust/CandidateEdge.cpp b/src/linclust/CandidateEdge.cpp index 8abf41713..2a3345e97 100644 --- a/src/linclust/CandidateEdge.cpp +++ b/src/linclust/CandidateEdge.cpp @@ -4,6 +4,8 @@ #include "FileUtil.h" #include "Util.h" +#include + #include #include #include @@ -29,9 +31,18 @@ void EdgeWriter::flush() { return; } if (file == NULL) { - file = fopen(path.c_str(), "wb"); + // Written under a per-worker temporary name and renamed on close. The + // output of this stage is named by bucket alone, so two workers that end + // up holding the same bucket -- which a lease expiry can still cause -- + // would otherwise interleave into one file, and the result would stay a + // whole number of records and pass every downstream integrity check. + // rename(2) is atomic, so the loser simply replaces the winner with an + // equally complete file. + tmpPath = path + ".w" + SSTR(getpid()); + file = fopen(tmpPath.c_str(), "wb"); if (file == NULL) { - Debug(Debug::ERROR) << "Cannot open edge file " << path << ": " << strerror(errno) << "\n"; + Debug(Debug::ERROR) << "Cannot open edge file " << tmpPath << ": " << strerror(errno) + << "\n"; EXIT(EXIT_FAILURE); } } @@ -64,17 +75,29 @@ void EdgeWriter::close() { // can tell "this partition was reduced and had nothing" from "this // partition was never reduced". if (fclose(file) != 0) { - Debug(Debug::ERROR) << "Cannot close edge file " << path << ": " << strerror(errno) << "\n"; + Debug(Debug::ERROR) << "Cannot close edge file " << tmpPath << ": " << strerror(errno) + << "\n"; EXIT(EXIT_FAILURE); } file = NULL; + if (rename(tmpPath.c_str(), path.c_str()) != 0) { + Debug(Debug::ERROR) << "Cannot rename " << tmpPath << " to " << path << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } } else { - FILE *empty = fopen(path.c_str(), "wb"); + const std::string tmp = path + ".w" + SSTR(getpid()); + FILE *empty = fopen(tmp.c_str(), "wb"); if (empty == NULL) { - Debug(Debug::ERROR) << "Cannot create edge file " << path << ": " << strerror(errno) << "\n"; + Debug(Debug::ERROR) << "Cannot create edge file " << tmp << ": " << strerror(errno) + << "\n"; + EXIT(EXIT_FAILURE); + } + if (fclose(empty) != 0 || rename(tmp.c_str(), path.c_str()) != 0) { + Debug(Debug::ERROR) << "Cannot publish empty edge file " << path << ": " + << strerror(errno) << "\n"; EXIT(EXIT_FAILURE); } - fclose(empty); } } @@ -122,7 +145,9 @@ void EdgeBucketWriter::flush(unsigned int bucket) { // Opened lazily: a worker whose partitions produced nothing for a bucket // should not cost a descriptor or an empty file. const std::string path = bucketDir(dir, bucket) + "/" + shardId + ".edges"; - files[bucket] = fopen(path.c_str(), "wb"); + // Append-and-close per flush, for the same reason as KmerBucketWriter: + // one descriptor per bucket would need up to 65536 of them. + files[bucket] = fopen(path.c_str(), "ab"); if (files[bucket] == NULL) { Debug(Debug::ERROR) << "Cannot open edge bucket " << path << ": " << strerror(errno) << "\n"; diff --git a/src/linclust/CandidateEdge.h b/src/linclust/CandidateEdge.h index fe16af5fd..abb55e490 100644 --- a/src/linclust/CandidateEdge.h +++ b/src/linclust/CandidateEdge.h @@ -10,7 +10,7 @@ // // Packed binary rather than a prefilter DB. A `pref` DB stores each hit as ASCII // with a per-representative index entry, and at 1e11 sequences the index alone is -// per-key state no single node can hold. 15 bytes per edge is also ~4x smaller +// per-key state no single node can hold. 17 bytes per edge is also ~4x smaller // than the text form, which matters directly: this is the largest intermediate // the pipeline writes. struct __attribute__((__packed__)) CandidateEdge { @@ -22,9 +22,19 @@ struct __attribute__((__packed__)) CandidateEdge { // Nucleotide strand. Stock carries it in bit 63 of the representative key, // which a 48-bit key has no room for. uint8_t reverseStrand; - // How many k-mers put this pair on this diagonal, saturating at 255. Stock's - // prefilter score, used downstream to rank and filter candidates. - uint8_t score; + // How many k-mers put this pair on this diagonal. Stock's prefilter score, + // and the value the align stage ranks diagonals by. + // + // 16 bits, not 8. The align stage picks a pair's diagonal by comparing these + // counts, so a ceiling low enough to be reached turns a comparison into a tie + // and hands the decision to a tie-break stock does not have. Pass 1 extracts + // 21 k-mers per sequence and never came close; pass 2 uses + // --kmer-per-seq-scale aa:0.100, so a long sequence contributes thousands, and + // at 255 this saturated on 199 of 4.9M records, which gave 23 pairs a + // different diagonal than stock and moved 31 of 1,000,000 sequences into a + // different cluster; widening it here took that residual to 8. The bound is kmersPerSeq(65535) x rounds ~= 26k, which + // fits 16 bits; stock accumulates in an int and has no ceiling at all. + uint16_t score; uint64_t getRep() const { return get(repBytes); } uint64_t getMember() const { return get(memberBytes); } @@ -74,6 +84,7 @@ class EdgeWriter { void flush(); std::string path; + std::string tmpPath; FILE *file; std::vector buffer; size_t bufferRecords; diff --git a/src/linclust/KmerPartition.cpp b/src/linclust/KmerPartition.cpp index 7dd3e8883..fa282a614 100644 --- a/src/linclust/KmerPartition.cpp +++ b/src/linclust/KmerPartition.cpp @@ -28,11 +28,29 @@ KmerPartitioner::KmerPartitioner(unsigned int partitionCount) : partitionCount(p namespace { +// Resident bytes the reduce needs per byte of k-mer bucket: 26/24 for the input +// array plus 20/24 for assignGroup's output array is 1.92, and the round-by-round +// candidate edges take the rest of the headroom. +const uint64_t REDUCE_MEMORY_FACTOR = 3; + +// More waves than this means the budget is wrong, not that the plan is clever. +const unsigned int MAX_SENSIBLE_WAVES = 64; + uint64_t divideRoundingUp(uint64_t value, uint64_t divisor) { return (value + divisor - 1) / divisor; } unsigned int roundUpToPowerOfTwo(uint64_t value) { + // Above 2^31 the shift below wraps to 0 and the loop never terminates, which + // an extreme --split-memory-limit or --scratch-budget can reach. Nothing here + // may legitimately exceed the 16-bit hash space anyway, so refuse early with a + // number the caller can act on. + if (value > 65536) { + Debug(Debug::ERROR) << "Derived a partition or wave count of " << value + << ", far past the 65536 the 16-bit k-mer hash can address. " + << "--split-memory-limit or --scratch-budget is implausibly small.\n"; + EXIT(EXIT_FAILURE); + } unsigned int result = 1; while (result < value) { result <<= 1; @@ -70,13 +88,36 @@ KmerShuffleSizing deriveKmerShuffleSizing(uint64_t sequenceCount, unsigned int k sizing.waveCount = roundUpToPowerOfTwo( std::max(divideRoundingUp(sizing.totalKmerBytes, available), 1)); } + // A budget only slightly above the persistent footprint leaves a sliver for + // the shuffle and derives a wave count in the hundreds -- a ~256x slowdown + // presented as a normal run. Refuse instead: at this point the budget is + // wrong, not the plan. + if (sizing.waveCount > MAX_SENSIBLE_WAVES) { + Debug(Debug::ERROR) << "A scratch budget of " << scratchBudgetBytes << " bytes leaves only " + << (scratchBudgetBytes > persistentBytes + ? scratchBudgetBytes - persistentBytes : 0) + << " bytes for " << sizing.totalKmerBytes << " bytes of k-mers, which " + << "needs " << sizing.waveCount << " extraction waves. Each wave " + << "re-scans every sequence, so this would run roughly " + << sizing.waveCount << "x slower than a single pass. Raise " + << "--scratch-budget above " << MAX_SENSIBLE_WAVES << " waves' worth.\n"; + EXIT(EXIT_FAILURE); + } sizing.bytesPerWave = divideRoundingUp(sizing.totalKmerBytes, sizing.waveCount); if (workerMemoryBytes == 0) { sizing.partitionCount = 1; } else { - sizing.partitionCount = - roundUpToPowerOfTwo(divideRoundingUp(sizing.bytesPerWave, workerMemoryBytes)); + // The reduce does not hold a partition at its on-disk size. It builds a + // KmerPosition array (26 B per 24 B record) and, alongside + // it, the KmerPosition array assignGroup writes into (20 B), + // then accumulates candidate edges across all rounds. Sizing P against the + // raw bucket bytes therefore overshot resident memory by ~1.9x before the + // edges were counted at all: with the workflow's default + // --split-memory-limit 0 on a 2 TB node at 1e11 that derived P = 32, a + // 1.58 TB partition and ~3 TB of arrays. + sizing.partitionCount = roundUpToPowerOfTwo( + divideRoundingUp(sizing.bytesPerWave * REDUCE_MEMORY_FACTOR, workerMemoryBytes)); } // Both are powers of two, so this makes the wave count a divisor of P. sizing.partitionCount = std::max(sizing.partitionCount, sizing.waveCount); @@ -145,8 +186,14 @@ void KmerBucketWriter::flush(unsigned int partition) { if (files[partition] == NULL) { // Opened lazily: with 8192 partitions and a sparse shard, most buckets // stay untouched and should not cost a file descriptor or an empty file. + // Opened in append mode and closed again after the write (see below). + // Holding one descriptor per partition for the life of the writer needs P + // of them -- 8192 at the 1e12 sizing -- against a soft limit + // FileUtil::fixRlimitNoFile only raises to 8192, so the last partitions + // would fail to open. Append is safe because this shard belongs to this + // worker alone. const std::string path = partitionDir(dir, partition) + "/" + shardId + ".kmers"; - files[partition] = fopen(path.c_str(), "wb"); + files[partition] = fopen(path.c_str(), "ab"); if (files[partition] == NULL) { Debug(Debug::ERROR) << "Cannot open bucket " << path << ": " << strerror(errno) << "\n"; EXIT(EXIT_FAILURE); @@ -160,6 +207,9 @@ void KmerBucketWriter::flush(unsigned int partition) { EXIT(EXIT_FAILURE); } buffer.clear(); + // Released immediately: see the note in flush() above. + fclose(files[partition]); + files[partition] = NULL; } void KmerBucketWriter::append(unsigned int partition, const KmerRecord &record) { diff --git a/src/linclust/alignparallel.cpp b/src/linclust/alignparallel.cpp index a689bddb0..897345fe0 100644 --- a/src/linclust/alignparallel.cpp +++ b/src/linclust/alignparallel.cpp @@ -112,7 +112,7 @@ size_t mergePairCopies(std::vector &edges) { edges[out] = edges[i]; edges[out].diagonal = bestDiagonal; edges[out].reverseStrand = bestStrand; - edges[out].score = static_cast(std::min(bestTotal, 255)); + edges[out].score = static_cast(std::min(bestTotal, 65535)); out++; i = j; } @@ -519,7 +519,7 @@ int alignparallel(int argc, const char **argv, const Command &command) { } // The greedy ranks by alignment score, not k-mer count. edges[e].score = - static_cast(std::min(aln.bitScore, 255)); + static_cast(std::min(aln.bitScore, 65535)); survives[e] = 1; continue; } @@ -578,7 +578,7 @@ int alignparallel(int argc, const char **argv, const Command &command) { continue; } edges[e].score = - static_cast(std::min(gapped.score1, 255)); + static_cast(std::min(gapped.score1, 65535)); survives[e] = 1; } } diff --git a/src/linclust/createrepdb.cpp b/src/linclust/createrepdb.cpp index 41ac171b9..8edba4bc2 100644 --- a/src/linclust/createrepdb.cpp +++ b/src/linclust/createrepdb.cpp @@ -157,6 +157,7 @@ CopyResult copyFlagged(const std::string &srcDb, const std::string &dstDb, const #pragma omp parallel num_threads(threads) { std::vector buf; + std::vector span; std::vector idxBuf; const uint64_t chunk = 4096; #pragma omp for schedule(dynamic, 1) @@ -166,11 +167,43 @@ CopyResult copyFlagged(const std::string &srcDb, const std::string &dstDb, const buf.resize(static_cast(bytes)); idxBuf.resize(static_cast(stop - start)); for (uint64_t i = start; i < stop; i++) { - readAt(srcData, buf.data() + (offsets[i] - offsets[start]), lengths[i], - srcEntries[i].offset, "source data"); idxBuf[i - start].offset = offsets[i]; idxBuf[i - start].length = lengths[i]; } + // Coalesced reads. Representatives are a *subset* of the source, so + // one pread per entry meant 353M random reads at 1e9 and made this + // stage 4x stock's createsubdb. Consecutive representatives are only a + // couple of sequence lengths apart, so a run of them is fetched in one + // read and the wanted pieces copied out. + // + // The run is closed when the bytes read would exceed twice the bytes + // actually wanted: coalescing across a sparse region reads the skipped + // sequences too, and without a cap a database whose representatives + // are thinly spread would read the whole file. That bounds the waste + // at 2x while still collapsing dense regions into single reads. + uint64_t i = start; + while (i < stop) { + const uint64_t from = srcEntries[i].offset; + uint64_t to = from + lengths[i]; + uint64_t wanted = lengths[i]; + uint64_t j = i + 1; + while (j < stop) { + const uint64_t end = srcEntries[j].offset + lengths[j]; + if (end - from > 2 * (wanted + lengths[j])) { + break; + } + to = end; + wanted += lengths[j]; + j++; + } + span.resize(static_cast(to - from)); + readAt(srcData, span.data(), static_cast(to - from), from, "source data"); + for (uint64_t k = i; k < j; k++) { + memcpy(buf.data() + (offsets[k] - offsets[start]), + span.data() + (srcEntries[k].offset - from), lengths[k]); + } + i = j; + } writeAt(dstData, buf.data(), static_cast(bytes), offsets[start], "database"); writeAt(dstIdx, idxBuf.data(), idxBuf.size() * sizeof(DenseIndex::Entry), DenseIndex::entryOffset(start), "index"); @@ -228,12 +261,27 @@ int createrepdb(int argc, const char **argv, const Command &command) { // subkey -> original key, dense in the sub-key space and in ascending order, // so the translation back is a sequential read rather than a lookup structure. const std::string mapFile = repDb + ".keymap"; - FILE *m = FileUtil::openAndDelete(mapFile.c_str(), "wb"); + const std::string keymapTmp = mapFile + ".tmp"; + FILE *m = FileUtil::openAndDelete(keymapTmp.c_str(), "wb"); if (fwrite(keyMap.data(), sizeof(uint64_t), keyMap.size(), m) != keyMap.size()) { - Debug(Debug::ERROR) << "Cannot write " << mapFile << "\n"; + Debug(Debug::ERROR) << "Cannot write " << keymapTmp << "\n"; + EXIT(EXIT_FAILURE); + } + if (fclose(m) != 0) { + + Debug(Debug::ERROR) << "Cannot close " << keymapTmp << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } - fclose(m); + + // Renamed only once complete: the pass-2 filter gate sizes itself from this + + // file's length, so a truncated one would silently be used as a short, + + // wrong sub-key to original-key map. + + FileUtil::move(keymapTmp.c_str(), mapFile.c_str()); const int dbType = FileUtil::parseDbType(seqDb.c_str()); FileUtil::writeFile(repDb + ".dbtype", reinterpret_cast(&dbType), diff --git a/src/linclust/greedycluster.cpp b/src/linclust/greedycluster.cpp index 0c4c7f4d9..5a24e4b88 100644 --- a/src/linclust/greedycluster.cpp +++ b/src/linclust/greedycluster.cpp @@ -56,6 +56,20 @@ #ifdef OPENMP #include + +// fwrite that fails loudly. Every caller here is building a final result file that +// the workflow renames into place on success; a short write that goes unnoticed +// becomes a truncated clustering the next restart treats as finished. +static void writeAllOrDie(const void *data, size_t bytes, FILE *file, const std::string &path) { + if (bytes == 0) { + return; + } + if (fwrite(data, 1, bytes, file) != bytes) { + Debug(Debug::ERROR) << "Cannot write " << bytes << " bytes to " << path << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } +} #endif namespace { @@ -253,13 +267,13 @@ int greedycluster(int argc, const char **argv, const Command &command) { assignedCount++; } if (buffer.size() > 32 * 1024 * 1024) { - fwrite(buffer.data(), 1, buffer.size(), out); + writeAllOrDie(buffer.data(), buffer.size(), out, outFile); buffer.clear(); } } } if (buffer.empty() == false) { - fwrite(buffer.data(), 1, buffer.size(), out); + writeAllOrDie(buffer.data(), buffer.size(), out, outFile); buffer.clear(); } diff --git a/src/linclust/kmermatcherparallel.cpp b/src/linclust/kmermatcherparallel.cpp index 833a0c654..4885b5a9b 100644 --- a/src/linclust/kmermatcherparallel.cpp +++ b/src/linclust/kmermatcherparallel.cpp @@ -26,6 +26,7 @@ #include "DBReader.h" #include "DenseIndex.h" #include "FileUtil.h" +#include "CandidateEdge.h" #include "KmerPartition.h" #include "NucleotideMatrix.h" #include "ParallelCoordination.h" @@ -42,6 +43,23 @@ namespace { +// Every file a sequence database actually occupies, not just its data file. +// createdbparallel writes the data, the dense index, the text index, the headers +// with their own two indices, .lookup and .source; sizing against the data file +// alone undercounts by ~1.64x on real data. +static uint64_t databaseFootprint(const std::string &db) { + static const char *suffixes[] = {"", ".index", ".index.bin", ".dbtype", ".lookup", ".source", + "_h", "_h.index", "_h.index.bin", "_h.dbtype"}; + uint64_t total = 0; + for (size_t i = 0; i < sizeof(suffixes) / sizeof(suffixes[0]); i++) { + const std::string path = db + suffixes[i]; + if (FileUtil::fileExists(path.c_str())) { + total += FileUtil::getFileSize(path); + } + } + return total; +} + // Work items are contiguous key ranges, so an item is also one contiguous read // of the data file. Their size is derived rather than fixed: a fixed size that // suits 1e12 sequences leaves a 1e6-sequence database with a single item, which @@ -246,11 +264,39 @@ int kmermatcherparallel(int argc, const char **argv, const Command &command) { const unsigned int kmersPerSequence = estimateKmersPerSequence(par, dbType, residues, info.entryCount); - // The sequence database shares the scratch budget with the k-mer buckets and - // outlives them, so it is not space the shuffle can use. + + // What the shuffle may NOT use, which is everything that is on the scratch + // filesystem while it runs and outlives it. + // + // This used to be `info.dataSize` -- the sequence *data file* alone. Measured + // on MGnify, the real database footprint is 1.64x that once the dense index, + // the text index, the headers, .lookup and .source are counted, and none of + // the pipeline's own output was counted at all. The result was that a 1e11 run + // handed the true 100 TB ceiling derived a single wave and peaked at ~186 TB. + // + // Two corrections. First, prefer a *measured* occupied figure passed by the + // caller (--scratch-used): the workflow knows what is already on disk, which + // is the only way pass 2 can account for what pass 1 left behind, and is why + // one budget knob can now size both passes. Second, reserve the candidate + // edges this stage's own reduce will write. Their volume is bounded, not + // guessed: a candidate edge needs a k-mer to have found it, so there can be no + // more edge records than k-mer records, and each is sizeof(CandidateEdge) + // against sizeof(KmerRecord). + const uint64_t totalKmerBytes = + static_cast(info.entryCount) * kmersPerSequence * sizeof(KmerRecord); + const uint64_t projectedEdgeBytes = + totalKmerBytes / sizeof(KmerRecord) * sizeof(CandidateEdge); + uint64_t occupied = par.scratchUsed > 0 ? static_cast(par.scratchUsed) + : databaseFootprint(seqDb); + const uint64_t persistentBytes = occupied + projectedEdgeBytes; + if (par.scratchBudget > 0) { + Debug(Debug::INFO) << "Scratch budget " << par.scratchBudget << " B; already occupied " + << occupied << " B; reserving " << projectedEdgeBytes + << " B for candidate edges\n"; + } const KmerShuffleSizing sizing = - deriveKmerShuffleSizing(info.entryCount, kmersPerSequence, par.scratchBudget, info.dataSize, - Util::computeMemory(par.splitMemoryLimit)); + deriveKmerShuffleSizing(info.entryCount, kmersPerSequence, par.scratchBudget, + persistentBytes, Util::computeMemory(par.splitMemoryLimit)); Debug(Debug::INFO) << "K-mer shuffle: " << sizing.partitionCount << " partitions, " << sizing.waveCount << " wave(s), " << sizing.totalKmerBytes << " k-mer bytes, " << sizing.bytesPerPartition << " bytes per partition\n"; diff --git a/src/linclust/kmerreduceparallel.cpp b/src/linclust/kmerreduceparallel.cpp index ae65c592a..4cbd59e04 100644 --- a/src/linclust/kmerreduceparallel.cpp +++ b/src/linclust/kmerreduceparallel.cpp @@ -82,6 +82,15 @@ size_t readPartitionAsPositions(const std::string &kmerDir, unsigned int partiti while (true) { const size_t got = fread(block.data(), sizeof(KmerRecord), blockRecords, file); if (got == 0) { + // A short read is EOF only if nothing went wrong. Treating an I/O + // error as EOF silently groups the partition with fewer k-mers, + // and the missing edges are indistinguishable from "this k-mer had + // no partner" -- a wrong answer with no diagnostic. + if (ferror(file)) { + Debug(Debug::ERROR) << "Cannot read k-mer bucket " << shards[i] << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } break; } if (filled + got > capacity) { @@ -211,7 +220,7 @@ void collectRoundEdges(KmerPosition *grouped, size_t writePos, b edge.setRep(static_cast(rep)); edge.setMember(static_cast(member)); edge.diagonal = static_cast(diagonal); - edge.score = static_cast(std::min(count, 255)); + edge.score = static_cast(std::min(count, 65535)); edges.push_back(edge); } i = j; @@ -224,7 +233,11 @@ bool compareEdge(const CandidateEdge &a, const CandidateEdge &b) { if (ra != rb) return ra < rb; const uint64_t ma = a.getMember(), mb = b.getMember(); if (ma != mb) return ma < mb; - return a.diagonal < b.diagonal; + if (a.diagonal != b.diagonal) return a.diagonal < b.diagonal; + // Strand belongs in the key: two opposite-strand edges on the same diagonal + // are different alignments, and collapsing them would sum their support. + // Only reachable for nucleotide input, which alignparallel rejects today. + return a.reverseStrand < b.reverseStrand; } @@ -317,9 +330,10 @@ uint64_t reducePartition(const std::string &kmerDir, for (size_t i = 0; i < edges.size(); i++) { if (unique > 0 && edges[i].getRep() == edges[unique - 1].getRep() && edges[i].getMember() == edges[unique - 1].getMember() && - edges[i].diagonal == edges[unique - 1].diagonal) { - edges[unique - 1].score = static_cast( - std::min(edges[unique - 1].score + edges[i].score, 255)); + edges[i].diagonal == edges[unique - 1].diagonal && + edges[i].reverseStrand == edges[unique - 1].reverseStrand) { + edges[unique - 1].score = static_cast( + std::min(edges[unique - 1].score + edges[i].score, 65535)); continue; } edges[unique] = edges[i]; @@ -433,7 +447,7 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { while (bucketCount < 65536 && info.dataSize / bucketCount > targetBucketBytes) { bucketCount *= 2; } - const uint64_t bucketSpan = (info.entryCount + bucketCount - 1) / bucketCount; + uint64_t bucketSpan = (info.entryCount + bucketCount - 1) / bucketCount; const std::string edgeManifest = reduceCoordDir + "/edge.info"; { @@ -449,6 +463,39 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { } lock.unlock(); } + // Re-read what the manifest actually says and refuse to disagree with it. + // + // Every worker derives bucketCount from Util::computeMemory(--split-memory-limit), + // which with the workflow's default of 0 is *this node's* RAM. A heterogeneous + // Slurm array, a restart on a different node, or a later wave would otherwise + // route edges into a different bucketing than the run began with -- writing + // into r directories createLayout never made. The map already does exactly + // this for shuffle.info. + { + unsigned int fileBucketCount = 0; + uint64_t fileBucketSpan = 0; + FILE *f = FileUtil::openFileOrDie(edgeManifest.c_str(), "r", true); + char name[64]; + size_t value; + while (fscanf(f, "%63s\t%zu\n", name, &value) == 2) { + const std::string key = name; + if (key == "bucketCount") fileBucketCount = static_cast(value); + else if (key == "bucketSpan") fileBucketSpan = value; + } + fclose(f); + if (fileBucketCount != bucketCount || fileBucketSpan != bucketSpan) { + Debug(Debug::ERROR) << "This worker derived " << bucketCount << " edge buckets of " + << bucketSpan << " keys, but " << edgeManifest << " says " + << fileBucketCount << " of " << fileBucketSpan + << ". The run was started with a different --split-memory-limit or " + << "on a node with different memory; edges would be routed into a " + << "different bucketing than the rest of the run. Re-run every " + << "worker of this stage with the same --split-memory-limit.\n"; + EXIT(EXIT_FAILURE); + } + bucketCount = fileBucketCount; + bucketSpan = fileBucketSpan; + } Debug(Debug::INFO) << "Writing edges into " << bucketCount << " representative-key buckets of " << bucketSpan << " keys\n"; @@ -487,13 +534,24 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { } } - if (waveCount > 1) { - // Waves exist only because scratch cannot hold the whole shuffle at once, - // so a reduced wave's buckets have to go before the next wave's map runs. + { + // The k-mer buckets are dead once their partitions have been reduced -- + // nothing downstream reads them again -- so they go here, at every wave + // count. Gating this on waveCount > 1, as it used to be, meant a + // single-wave run kept the whole shuffle on disk for the rest of the job, + // including all of pass 2: measured at 100M, kmer1 (49.8 GB) and kmer2 + // (52.9 GB) were both resident at peak, and the shuffle is the largest + // intermediate the pipeline writes. + // // Safe here because drain() returns only once every partition of the wave // is recorded done, which means no worker is still reading one. Several // workers reach this together, so a file another already unlinked is not // an error. + // Only the worker that recorded the last completion deletes. drain() + // returns true for every worker that observes the queue finished, and a + // worker whose lease lapsed could still be inside reducePartition; letting + // them all unlink races that reader. Heartbeats make the lapse unlikely, + // this makes the deletion single-writer regardless. for (unsigned int p = waveFrom; p < waveTo; p++) { const std::vector shards = KmerBucketReader::shardFiles(kmerDir, p); for (size_t i = 0; i < shards.size(); i++) { @@ -504,7 +562,8 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { } } } - Debug(Debug::INFO) << "Removed the k-mer buckets of wave " << par.kmerWave << "\n"; + Debug(Debug::INFO) << "Removed the consumed k-mer buckets" + << (waveCount > 1 ? " of wave " + SSTR(par.kmerWave) : "") << "\n"; } Debug(Debug::INFO) << "Worker " << workerId << " wrote " << edgeCount diff --git a/src/linclust/mergeclusterparallel.cpp b/src/linclust/mergeclusterparallel.cpp index e8cb217a7..bb53e42cf 100644 --- a/src/linclust/mergeclusterparallel.cpp +++ b/src/linclust/mergeclusterparallel.cpp @@ -35,6 +35,20 @@ #include #include +// fwrite that fails loudly. Every caller here is building a final result file that +// the workflow renames into place on success; a short write that goes unnoticed +// becomes a truncated clustering the next restart treats as finished. +static void writeAllOrDie(const void *data, size_t bytes, FILE *file, const std::string &path) { + if (bytes == 0) { + return; + } + if (fwrite(data, 1, bytes, file) != bytes) { + Debug(Debug::ERROR) << "Cannot write " << bytes << " bytes to " << path << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } +} + namespace { // (key, value) as written to a bucket: for the later clustering the key is the @@ -206,7 +220,7 @@ void compose(const std::string &earlier, const std::string &later, const std::st const uint64_t mapped = remap[static_cast(rep - lo)]; appendPair(buffer, mapped == INVALID ? rep : mapped, earlierPairs[i].value); if (buffer.size() > 32 * 1024 * 1024) { - fwrite(buffer.data(), 1, buffer.size(), result); + writeAllOrDie(buffer.data(), buffer.size(), result, out); buffer.clear(); } } @@ -214,7 +228,7 @@ void compose(const std::string &earlier, const std::string &later, const std::st FileUtil::remove(BucketWriter::path(tmpPrefix + ".earlier", b).c_str()); } if (buffer.empty() == false) { - fwrite(buffer.data(), 1, buffer.size(), result); + writeAllOrDie(buffer.data(), buffer.size(), result, out); } if (fclose(result) != 0) { Debug(Debug::ERROR) << "Cannot close " << out << "\n"; diff --git a/src/linclust/translatecluster.cpp b/src/linclust/translatecluster.cpp index 4a1c3a7b1..eca1c9636 100644 --- a/src/linclust/translatecluster.cpp +++ b/src/linclust/translatecluster.cpp @@ -30,6 +30,20 @@ #include #include +// fwrite that fails loudly. Every caller here is building a final result file that +// the workflow renames into place on success; a short write that goes unnoticed +// becomes a truncated clustering the next restart treats as finished. +static void writeAllOrDie(const void *data, size_t bytes, FILE *file, const std::string &path) { + if (bytes == 0) { + return; + } + if (fwrite(data, 1, bytes, file) != bytes) { + Debug(Debug::ERROR) << "Cannot write " << bytes << " bytes to " << path << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } +} + namespace { struct __attribute__((__packed__)) Pair { @@ -201,6 +215,11 @@ int translatecluster(int argc, const char **argv, const Command &command) { const std::vector slice = mapSlice(mapFd, lo, hi); const std::vector pairs = Buckets::read(tmpA, b); for (size_t i = 0; i < pairs.size(); i++) { + if (pairs[i].key < lo || pairs[i].key - lo >= slice.size()) { + Debug(Debug::ERROR) << "Sub-key " << pairs[i].key << " is outside the key map " + << mapFile << ", which holds " << subCount << " keys\n"; + EXIT(EXIT_FAILURE); + } const uint64_t origMember = slice[static_cast(pairs[i].key - lo)]; // Re-bucket on the representative sub-key for the second pass. byRep.append(static_cast(pairs[i].other / span), pairs[i].other, @@ -222,16 +241,21 @@ int translatecluster(int argc, const char **argv, const Command &command) { const std::vector slice = mapSlice(mapFd, lo, hi); const std::vector pairs = Buckets::read(tmpB, b); for (size_t i = 0; i < pairs.size(); i++) { + if (pairs[i].key < lo || pairs[i].key - lo >= slice.size()) { + Debug(Debug::ERROR) << "Sub-key " << pairs[i].key << " is outside the key map " + << mapFile << ", which holds " << subCount << " keys\n"; + EXIT(EXIT_FAILURE); + } appendPair(buffer, slice[static_cast(pairs[i].key - lo)], pairs[i].other); written++; if (buffer.size() > 32 * 1024 * 1024) { - fwrite(buffer.data(), 1, buffer.size(), out); + writeAllOrDie(buffer.data(), buffer.size(), out, outTsv); buffer.clear(); } } FileUtil::remove(Buckets::path(tmpB, b).c_str()); } - if (buffer.empty() == false) fwrite(buffer.data(), 1, buffer.size(), out); + if (buffer.empty() == false) writeAllOrDie(buffer.data(), buffer.size(), out, outTsv); if (fclose(out) != 0) { Debug(Debug::ERROR) << "Cannot close " << outTsv << "\n"; EXIT(EXIT_FAILURE); diff --git a/src/linclust/translatekeys.cpp b/src/linclust/translatekeys.cpp index affbe7975..9b13046c7 100644 --- a/src/linclust/translatekeys.cpp +++ b/src/linclust/translatekeys.cpp @@ -36,6 +36,24 @@ #include #include +#ifdef OPENMP +#include +#endif + +// fwrite that fails loudly. Every caller here is building a final result file that +// the workflow renames into place on success; a short write that goes unnoticed +// becomes a truncated clustering the next restart treats as finished. +static void writeAllOrDie(const void *data, size_t bytes, FILE *file, const std::string &path) { + if (bytes == 0) { + return; + } + if (fwrite(data, 1, bytes, file) != bytes) { + Debug(Debug::ERROR) << "Cannot write " << bytes << " bytes to " << path << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } +} + namespace { // Spill file holding (key, payload) records whose payload is a key too. @@ -216,8 +234,12 @@ class KeyBuckets { // detects by finding an empty accession. class LookupCursor { public: - explicit LookupCursor(const std::string &path) : line(NULL), cap(0), pending(false), pendingKey(0) { + explicit LookupCursor(const std::string &path, uint64_t startOffset = 0) + : line(NULL), cap(0), pending(false), pendingKey(0) { file = FileUtil::openFileOrDie(path.c_str(), "r", true); + if (startOffset > 0) { + fseeko(file, static_cast(startOffset), SEEK_SET); + } } ~LookupCursor() { free(line); @@ -256,8 +278,44 @@ class LookupCursor { std::string pendingAccession; }; +// Byte offset in the lookup where each bucket's key range begins. +// +// The lookup is variable-width text, so a bucket cannot seek to its own keys -- +// which is why the three passes below used to share one forward cursor and run +// strictly one bucket at a time. One sequential scan recording `buckets` offsets +// (a few KB) makes every bucket independent, and the passes become parallel. +static std::vector indexLookup(const std::string &path, uint64_t span, + unsigned int buckets) { + std::vector at(buckets, UINT64_MAX); + FILE *f = FileUtil::openFileOrDie(path.c_str(), "r", true); + char *line = NULL; + size_t cap = 0; + ssize_t len; + uint64_t offset = 0; + while ((len = getline(&line, &cap, f)) > 0) { + const uint64_t key = strtoull(line, NULL, 10); + const unsigned int b = static_cast(key / span); + if (b < buckets && at[b] == UINT64_MAX) { + at[b] = offset; + } + offset += static_cast(len); + } + free(line); + fclose(f); + return at; +} + const std::string &accessionOf(const std::vector &slice, uint64_t key, uint64_t lo, const std::string &lookupFile) { + // Checked before indexing, not after. A clustering naming a key the lookup + // does not have is exactly the case the message below describes, and reading + // slice[key - lo] first would go out of bounds before it could be printed. + if (key < lo || key - lo >= slice.size()) { + Debug(Debug::ERROR) << "Key " << key << " of the clustering is outside the range the " + << "lookup " << lookupFile << " covers. The clustering and the lookup " + << "are from different databases.\n"; + EXIT(EXIT_FAILURE); + } const std::string &name = slice[static_cast(key - lo)]; if (name.empty()) { Debug(Debug::ERROR) << "Key " << key << " of the clustering has no entry in " << lookupFile @@ -276,6 +334,7 @@ int translatekeys(int argc, const char **argv, const Command &command) { const std::string inTsv = par.db1; const std::string lookupFile = par.db2; const std::string outTsv = par.db3; + const int threads = std::max(1, par.threads); // Sized so one slice of accessions fits the memory budget. An accession is // ~30 bytes plus the std::string that holds it, so 64 bytes per key is a safe @@ -321,6 +380,13 @@ int translatekeys(int argc, const char **argv, const Command &command) { if (tab == NULL) continue; const uint64_t rep = strtoull(line, NULL, 10); const uint64_t member = strtoull(tab + 1, NULL, 10); + if (member / span >= buckets || rep / span >= buckets) { + Debug(Debug::ERROR) << "Clustering names key " << std::max(member, rep) + << ", beyond the " << keyCount << " keys in " << lookupFile + << ". The clustering and the lookup are from different " + << "databases.\n"; + EXIT(EXIT_FAILURE); + } byMember.append(static_cast(member / span), member, rep); } free(line); @@ -328,57 +394,115 @@ int translatekeys(int argc, const char **argv, const Command &command) { } // Pass 2: translate the member, re-bucket on the representative. + // + // Parallel over buckets, each thread seeking straight to its own key range. + // Its output goes to a thread-private set of representative buckets, so no + // two threads share a writer; pass 3 reads all shards of a bucket. + const std::vector lookupAt = indexLookup(lookupFile, span, buckets); { - LookupCursor cursor(lookupFile); - TextBuckets byRep(tmpB, buckets); - std::vector slice; - for (unsigned int b = 0; b < buckets; b++) { - const uint64_t lo = b * span; - if (lo >= keyCount) break; - const uint64_t hi = std::min(lo + span, keyCount); - cursor.slice(lo, hi, slice); - const std::vector pairs = KeyBuckets::read(tmpA, b); - for (size_t i = 0; i < pairs.size(); i++) { - const std::string &name = accessionOf(slice, pairs[i].key, lo, lookupFile); - byRep.append(static_cast(pairs[i].other / span), pairs[i].other, - name.c_str(), name.size()); + std::vector writers(threads, NULL); + for (int t = 0; t < threads; t++) { + writers[t] = new TextBuckets(tmpB + ".t" + SSTR(t), buckets); + } +#pragma omp parallel num_threads(threads) + { + int tid = 0; +#ifdef OPENMP + tid = omp_get_thread_num(); +#endif + std::vector slice; +#pragma omp for schedule(dynamic, 1) + for (int64_t bi = 0; bi < static_cast(buckets); bi++) { + const unsigned int b = static_cast(bi); + const uint64_t lo = b * span; + if (lo >= keyCount || lookupAt[b] == UINT64_MAX) { + continue; + } + const uint64_t hi = std::min(lo + span, keyCount); + LookupCursor cursor(lookupFile, lookupAt[b]); + cursor.slice(lo, hi, slice); + const std::vector pairs = KeyBuckets::read(tmpA, b); + for (size_t i = 0; i < pairs.size(); i++) { + const std::string &name = accessionOf(slice, pairs[i].key, lo, lookupFile); + writers[tid]->append(static_cast(pairs[i].other / span), + pairs[i].other, name.c_str(), name.size()); + } + FileUtil::remove(KeyBuckets::path(tmpA, b).c_str()); } - FileUtil::remove(KeyBuckets::path(tmpA, b).c_str()); + } + for (int t = 0; t < threads; t++) { + delete writers[t]; } } // Pass 3: translate the representative and emit. - LookupCursor cursor(lookupFile); - FILE *out = FileUtil::openAndDelete(outTsv.c_str(), "w"); - std::string buffer; - buffer.reserve(64 * 1024 * 1024); - std::vector slice; - std::vector > entries; + // + // Also parallel over buckets, each writing its own piece; the pieces are then + // concatenated in bucket order, so the output is exactly what the sequential + // version produced. uint64_t written = 0; - for (unsigned int b = 0; b < buckets; b++) { - const uint64_t lo = b * span; - if (lo >= keyCount) break; - const uint64_t hi = std::min(lo + span, keyCount); - cursor.slice(lo, hi, slice); - TextBuckets::read(tmpB, b, entries); - for (size_t i = 0; i < entries.size(); i++) { - const std::string &repName = accessionOf(slice, entries[i].first, lo, lookupFile); - buffer.append(repName); - buffer.push_back('\t'); - buffer.append(entries[i].second); - buffer.push_back('\n'); - written++; - if (buffer.size() > 32 * 1024 * 1024) { - fwrite(buffer.data(), 1, buffer.size(), out); + { +#pragma omp parallel num_threads(threads) reduction(+ : written) + { + std::vector slice; + std::vector > entries; + std::string buffer; +#pragma omp for schedule(dynamic, 1) + for (int64_t bi = 0; bi < static_cast(buckets); bi++) { + const unsigned int b = static_cast(bi); + const uint64_t lo = b * span; + if (lo >= keyCount || lookupAt[b] == UINT64_MAX) { + continue; + } + const uint64_t hi = std::min(lo + span, keyCount); + LookupCursor cursor(lookupFile, lookupAt[b]); + cursor.slice(lo, hi, slice); buffer.clear(); + for (int t = 0; t < threads; t++) { + TextBuckets::read(tmpB + ".t" + SSTR(t), b, entries); + for (size_t i = 0; i < entries.size(); i++) { + const std::string &repName = + accessionOf(slice, entries[i].first, lo, lookupFile); + buffer.append(repName); + buffer.push_back('\t'); + buffer.append(entries[i].second); + buffer.push_back('\n'); + written++; + } + // A (thread, bucket) shard exists only if that thread wrote + // to that bucket, which is sparse. + const std::string shard = TextBuckets::path(tmpB + ".t" + SSTR(t), b); + if (FileUtil::fileExists(shard.c_str())) { + FileUtil::remove(shard.c_str()); + } + } + FILE *piece = FileUtil::openAndDelete((outTsv + ".p" + SSTR(b)).c_str(), "w"); + writeAllOrDie(buffer.data(), buffer.size(), piece, outTsv); + if (fclose(piece) != 0) { + Debug(Debug::ERROR) << "Cannot close " << outTsv << ".p" << b << "\n"; + EXIT(EXIT_FAILURE); + } } } - FileUtil::remove(TextBuckets::path(tmpB, b).c_str()); - } - if (buffer.empty() == false) fwrite(buffer.data(), 1, buffer.size(), out); - if (fclose(out) != 0) { - Debug(Debug::ERROR) << "Cannot close " << outTsv << "\n"; - EXIT(EXIT_FAILURE); + FILE *out = FileUtil::openAndDelete(outTsv.c_str(), "w"); + std::vector copy(8 << 20); + for (unsigned int b = 0; b < buckets; b++) { + const std::string piecePath = outTsv + ".p" + SSTR(b); + if (FileUtil::fileExists(piecePath.c_str()) == false) { + continue; + } + FILE *piece = FileUtil::openFileOrDie(piecePath.c_str(), "rb", true); + size_t got; + while ((got = fread(copy.data(), 1, copy.size(), piece)) > 0) { + writeAllOrDie(copy.data(), got, out, outTsv); + } + fclose(piece); + FileUtil::remove(piecePath.c_str()); + } + if (fclose(out) != 0) { + Debug(Debug::ERROR) << "Cannot close " << outTsv << "\n"; + EXIT(EXIT_FAILURE); + } } Debug(Debug::INFO) << "Translated " << written << " assignments into " << outTsv << "\n"; diff --git a/src/test/TestKmerPartition.cpp b/src/test/TestKmerPartition.cpp index 87db7c8f3..add7b9d82 100644 --- a/src/test/TestKmerPartition.cpp +++ b/src/test/TestKmerPartition.cpp @@ -300,13 +300,18 @@ static void testShuffleSizing() { check(small.waveCount == 1, "100B fits the 100 TB budget in a single wave"); check(small.bytesPerWave <= 100 * TB - 33 * TB, "100B peak k-mer bytes stay inside the budget after the database and edges"); - check(small.partitionCount == 1024, "100B derives P = 1024"); + check(small.partitionCount == 4096, "100B derives P = 4096"); + // P bounds the reduce's RESIDENT set, which is ~1.9x the bucket's on-disk + // bytes before candidate edges; the factor of 3 is what keeps a partition + // inside a worker rather than 1.9x outside it. + check(small.bytesPerPartition * 3 <= 64 * GB, "100B partitions fit worker memory when expanded"); check(small.bytesPerPartition <= 64 * GB, "100B buckets fit the per-worker memory"); // 1T: 1e12 sequences, no hard ceiling, so only per-worker memory sets P. KmerShuffleSizing large = deriveKmerShuffleSizing(1000000000000ULL, 21, 0, 0, 64 * GB); check(large.waveCount == 1, "an unlimited budget means a single wave"); - check(large.partitionCount == 8192, "1T derives P = 8192"); + check(large.partitionCount == 32768, "1T derives P = 32768"); + check(large.bytesPerPartition * 3 <= 64 * GB, "1T partitions fit worker memory when expanded"); check(large.bytesPerPartition <= 64 * GB, "1T buckets fit the per-worker memory"); check(large.partitionCount > small.partitionCount, "the two target scales genuinely want different P, which is why it is derived"); @@ -325,7 +330,9 @@ static void testShuffleSizing() { // -- and peak scratch only really 1/W of the whole -- if W divides P. Both // being powers of two is how that is guaranteed. bool wavesDivide = true; - for (uint64_t budget = 4; budget <= 512; budget *= 2) { + // From 8 TB up: below that, 504 TB of k-mers needs more than the 64 waves + // deriveKmerShuffleSizing now refuses as a misconfigured budget. + for (uint64_t budget = 16; budget <= 512; budget *= 2) { // Sweep budgets from far below the k-mer volume to above it. Per-worker // memory is large so nothing but the wave count can raise P, which is the // case where the two used to disagree. From 2c31978b67880e228b980012897c712db68eebbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Mon, 3 Aug 2026 08:29:33 +0000 Subject: [PATCH 10/27] Fix handling of duplicate edges across blocks. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Björn Buschkämper --- src/commons/ParallelCoordination.cpp | 34 +++++++++++ src/commons/ParallelCoordination.h | 11 ++++ src/linclust/CandidateEdge.cpp | 83 ++++++++++++++++++++++++++- src/linclust/CandidateEdge.h | 45 +++++++++++++++ src/linclust/alignparallel.cpp | 50 +++++++++------- src/linclust/kmerreduceparallel.cpp | 5 ++ src/linclust/translatekeys.cpp | 80 ++++++++++++++++++-------- src/test/TestParallelCoordination.cpp | 31 ++++++++++ 8 files changed, 292 insertions(+), 47 deletions(-) diff --git a/src/commons/ParallelCoordination.cpp b/src/commons/ParallelCoordination.cpp index ef5aface6..4bedf8684 100644 --- a/src/commons/ParallelCoordination.cpp +++ b/src/commons/ParallelCoordination.cpp @@ -390,6 +390,40 @@ bool WorkQueue::allDone() { return getDoneCount() >= itemCount; } +bool WorkQueue::readCompletedWorkers(const std::string &path, std::vector &workers) { + workers.clear(); + const int fd = open(path.c_str(), O_RDONLY); + if (fd < 0) { + return false; + } + Header header; + if (preadFully(fd, &header, sizeof(Header), 0) != static_cast(sizeof(Header))) { + close(fd); + Debug(Debug::ERROR) << "Cannot read the header of work queue " << path << "\n"; + EXIT(EXIT_FAILURE); + } + if (header.magic != MAGIC || header.version != VERSION) { + close(fd); + Debug(Debug::ERROR) << "File " << path << " is not a work queue\n"; + EXIT(EXIT_FAILURE); + } + workers.assign(static_cast(header.itemCount), -1); + for (uint64_t i = 0; i < header.itemCount; i++) { + Record record; + if (preadFully(fd, &record, sizeof(Record), recordOffset(static_cast(i))) != + static_cast(sizeof(Record))) { + close(fd); + Debug(Debug::ERROR) << "Cannot read record " << i << " of work queue " << path << "\n"; + EXIT(EXIT_FAILURE); + } + if (record.state == DONE) { + workers[static_cast(i)] = static_cast(record.worker); + } + } + close(fd); + return true; +} + bool WorkQueue::awaitAll(unsigned int pollSeconds, unsigned int stallSeconds) { int64_t lastDone = -1; int64_t lastProgress = nowSeconds(); diff --git a/src/commons/ParallelCoordination.h b/src/commons/ParallelCoordination.h index f5f930676..281a48c0b 100644 --- a/src/commons/ParallelCoordination.h +++ b/src/commons/ParallelCoordination.h @@ -8,6 +8,7 @@ #include #include #include +#include // Shared-filesystem coordination for multi-node MMseqs2 stages. // @@ -128,6 +129,16 @@ class WorkQueue { int64_t getDoneCount(); bool allDone(); + // Reads a finished queue's completion records without opening it as a queue, + // which would create it and needs an itemCount the caller does not know. + // + // workers[i] is the worker that recorded item i DONE, or -1 if it never was. + // Exactly one worker is named per completed item -- complete() is idempotent + // and keeps the first recorded worker -- which makes this usable as the + // authority on whose output for an item counts, when an item may have been + // run more than once. Returns false if the queue file does not exist. + static bool readCompletedWorkers(const std::string &path, std::vector &workers); + // True while some item is held under an unexpired lease. bool hasLiveClaim(); diff --git a/src/linclust/CandidateEdge.cpp b/src/linclust/CandidateEdge.cpp index 2a3345e97..75934a9d0 100644 --- a/src/linclust/CandidateEdge.cpp +++ b/src/linclust/CandidateEdge.cpp @@ -125,7 +125,8 @@ void EdgeBucketWriter::createLayout(const std::string &dir, unsigned int bucketC EdgeBucketWriter::EdgeBucketWriter(const std::string &dir, unsigned int bucketCount, const std::string &shardId, size_t bufferBudgetBytes) - : dir(dir), shardId(shardId), bucketCount(bucketCount), edgeCount(0), closed(false) { + : dir(dir), shardId(shardId), bucketCount(bucketCount), edgeCount(0), closed(false), + currentPartition(0), currentWorker(-1) { const size_t perBucket = bufferBudgetBytes / (bucketCount * sizeof(CandidateEdge)); edgesPerBuffer = std::max(perBucket, 64); buffers.resize(bucketCount); @@ -154,6 +155,19 @@ void EdgeBucketWriter::flush(unsigned int bucket) { EXIT(EXIT_FAILURE); } } + // Header then records, as one write each. The header names the producer so a + // crashed worker's superseded copy can be told apart from a second partition + // that legitimately produced the same edges (see EdgeBlockHeader). + EdgeBlockHeader header; + header.magic = EdgeBlockHeader::MAGIC; + header.partition = currentPartition; + header.worker = static_cast(currentWorker < 0 ? 0 : currentWorker); + header.recordCount = static_cast(buffer.size()); + if (fwrite(&header, sizeof(EdgeBlockHeader), 1, files[bucket]) != 1) { + Debug(Debug::ERROR) << "Cannot write the block header of bucket " << bucket << " of " << dir + << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } if (fwrite(buffer.data(), sizeof(CandidateEdge), buffer.size(), files[bucket]) != buffer.size()) { Debug(Debug::ERROR) << "Cannot write " << buffer.size() << " edges to bucket " << bucket << " of " << dir << ": " << strerror(errno) << "\n"; @@ -162,6 +176,18 @@ void EdgeBucketWriter::flush(unsigned int bucket) { buffer.clear(); } +void EdgeBucketWriter::beginPartition(unsigned int partition, int64_t worker) { + if (currentWorker >= 0 && (partition != currentPartition || worker != currentWorker)) { + // A block must belong to exactly one partition, or the filter cannot + // decide it. Anything still buffered is the previous one's. + for (unsigned int b = 0; b < bucketCount; b++) { + flush(b); + } + } + currentPartition = partition; + currentWorker = worker; +} + void EdgeBucketWriter::append(unsigned int bucket, const CandidateEdge &edge) { buffers[bucket].push_back(edge); if (buffers[bucket].size() >= edgesPerBuffer) { @@ -215,3 +241,58 @@ std::vector EdgeBucketWriter::shardFiles(const std::string &dir, un std::sort(shards.begin(), shards.end()); return shards; } + +size_t EdgeBucketReader::readShard(const std::string &path, const std::vector &authority, + std::vector &out) { + const size_t bytes = FileUtil::getFileSize(path); + if (bytes == 0) { + return 0; + } + FILE *file = FileUtil::openFileOrDie(path.c_str(), "rb", true); + size_t offset = 0; + size_t kept = 0; + while (offset + sizeof(EdgeBlockHeader) <= bytes) { + EdgeBlockHeader header; + if (fread(&header, sizeof(EdgeBlockHeader), 1, file) != 1) { + break; + } + offset += sizeof(EdgeBlockHeader); + const size_t blockBytes = static_cast(header.recordCount) * sizeof(CandidateEdge); + // A worker killed mid-write leaves a partial block, always at the end of + // its own shard: it appends, and a restarted worker takes a new id and so + // a new shard. Stopping here discards exactly that tail. The partition it + // belonged to is redone by another worker, whose copy is complete. + if (header.magic != EdgeBlockHeader::MAGIC || offset + blockBytes > bytes) { + Debug(Debug::WARNING) << "Edge shard " << path << " ends in a partial block at byte " + << offset << "; it was written by an interrupted worker and the " + << "partition it held was redone.\n"; + break; + } + const bool wanted = + authority.empty() || header.partition >= authority.size() || + authority[header.partition] == static_cast(header.worker); + if (wanted == false) { + // A superseded copy: this worker did not record the partition done, so + // another redid it and its edges are the ones that count. + if (fseek(file, static_cast(blockBytes), SEEK_CUR) != 0) { + Debug(Debug::ERROR) << "Cannot skip a superseded block in " << path << "\n"; + EXIT(EXIT_FAILURE); + } + offset += blockBytes; + continue; + } + if (header.recordCount > 0) { + const size_t at = out.size(); + out.resize(at + header.recordCount); + if (fread(out.data() + at, sizeof(CandidateEdge), header.recordCount, file) != + header.recordCount) { + Debug(Debug::ERROR) << "Cannot read a block of edge shard " << path << "\n"; + EXIT(EXIT_FAILURE); + } + kept += header.recordCount; + } + offset += blockBytes; + } + fclose(file); + return kept; +} diff --git a/src/linclust/CandidateEdge.h b/src/linclust/CandidateEdge.h index abb55e490..a709d130f 100644 --- a/src/linclust/CandidateEdge.h +++ b/src/linclust/CandidateEdge.h @@ -110,6 +110,31 @@ class EdgeWriter { // accumulation stock does in its global merge can be reproduced exactly, // instead of approximated per partition. // +// Framing for one flushed run of edges, so a record can be traced to the k-mer +// partition and the worker that produced it. +// +// This is what makes the reduce idempotent across a crash. A worker that flushes +// a partition's edges and dies before the queue records the item leaves that data +// on disk; the item is then re-claimed and redone by another worker, whose edges +// land in *its* shard. Both copies are present, and the align stage sums the +// support of matching (pair, diagonal) records, so the redone pair would count +// twice and can win a diagonal it should not. +// +// The duplicates cannot be recognised from the records alone: unlike a k-mer +// record, where (kmer, id, pos) names one occurrence, two different partitions +// may legitimately emit byte-identical edges that must both be summed. Naming the +// producer in the frame is what separates "the same partition written twice" from +// "two partitions that agree". +struct __attribute__((__packed__)) EdgeBlockHeader { + // Distinguishes a real header from the tail of a torn write. + uint32_t magic; + uint32_t partition; + uint32_t worker; + uint32_t recordCount; + + static const uint32_t MAGIC = 0x45444745; // "EDGE" +}; + // One file per (worker, bucket), like the k-mer shards, so no locking is needed. class EdgeBucketWriter { public: @@ -117,6 +142,11 @@ class EdgeBucketWriter { size_t bufferBudgetBytes = 256 * 1024 * 1024); ~EdgeBucketWriter(); + // Names the partition every subsequent append belongs to, and the worker + // doing the work. Flushes anything still buffered for the previous partition + // first, so a block never spans two of them. + void beginPartition(unsigned int partition, int64_t worker); + void append(unsigned int bucket, const CandidateEdge &edge); // Pushes buffered edges to the OS. Call before marking a work item done, for // the same reason the k-mer writer does. @@ -143,6 +173,21 @@ class EdgeBucketWriter { std::vector files; uint64_t edgeCount; bool closed; + unsigned int currentPartition; + int64_t currentWorker; +}; + +// Reads the blocks EdgeBucketWriter wrote, keeping only those whose producer the +// reduce's work queue recorded as the one that completed the partition. +// +// `authority[partition]` is that worker (see WorkQueue::readCompletedWorkers). A +// block naming any other worker is a dead worker's copy of an item that was +// redone, and is skipped. An empty authority disables filtering, for callers that +// have no queue to consult. +class EdgeBucketReader { +public: + static size_t readShard(const std::string &path, const std::vector &authority, + std::vector &out); }; #endif diff --git a/src/linclust/alignparallel.cpp b/src/linclust/alignparallel.cpp index 897345fe0..8beceaec5 100644 --- a/src/linclust/alignparallel.cpp +++ b/src/linclust/alignparallel.cpp @@ -120,29 +120,36 @@ size_t mergePairCopies(std::vector &edges) { return out; } +// Which worker's edges count for each k-mer partition. +// +// The reduce's work queue is the authority: complete() keeps the first worker to +// record an item done, so exactly one is named per partition, and any block from +// another worker is a copy left by one that died before recording it. One queue +// per extraction wave, and wave w holds the partitions after all earlier waves', +// so appending them in wave order indexes by partition directly. +std::vector readReduceAuthority(const std::string &edgeDir) { + std::vector authority; + for (unsigned int wave = 0;; wave++) { + std::vector workers; + const std::string path = edgeDir + "/coord/reduce." + SSTR(wave) + ".queue"; + if (WorkQueue::readCompletedWorkers(path, workers) == false) { + break; + } + authority.insert(authority.end(), workers.begin(), workers.end()); + } + if (authority.empty()) { + Debug(Debug::WARNING) << "No reduce work queue under " << edgeDir + << "/coord; cannot tell a crashed worker's superseded edges from a " + << "second partition's. Duplicate support would be summed.\n"; + } + return authority; +} + size_t readBucket(const std::string &edgeDir, unsigned int bucket, - std::vector &out) { + const std::vector &authority, std::vector &out) { const std::vector shards = EdgeBucketWriter::shardFiles(edgeDir, bucket); for (size_t i = 0; i < shards.size(); i++) { - const size_t bytes = FileUtil::getFileSize(shards[i]); - if (bytes % sizeof(CandidateEdge) != 0) { - Debug(Debug::ERROR) << "Edge shard " << shards[i] << " is " << bytes - << " bytes, not a whole number of edges. It was probably written " - << "by an interrupted worker.\n"; - EXIT(EXIT_FAILURE); - } - const size_t count = bytes / sizeof(CandidateEdge); - if (count == 0) { - continue; - } - FILE *file = FileUtil::openFileOrDie(shards[i].c_str(), "rb", true); - const size_t offset = out.size(); - out.resize(offset + count); - if (fread(out.data() + offset, sizeof(CandidateEdge), count, file) != count) { - Debug(Debug::ERROR) << "Cannot read edge shard " << shards[i] << "\n"; - EXIT(EXIT_FAILURE); - } - fclose(file); + EdgeBucketReader::readShard(shards[i], authority, out); } return out.size(); } @@ -382,12 +389,13 @@ int alignparallel(int argc, const char **argv, const Command &command) { PartitionSequences sequences(seqDb); uint64_t survivorCount = 0; + const std::vector reduceAuthority = readReduceAuthority(edgeDir); { WorkQueue queue(coordDir + "/align.queue", static_cast(bucketCount)); const bool finished = queue.drain(workerId, [&](size_t bucket) { std::vector edges; - readBucket(edgeDir, static_cast(bucket), edges); + readBucket(edgeDir, static_cast(bucket), reduceAuthority, edges); const size_t raw = edges.size(); if (raw == 0) { EdgeWriter empty(EdgeWriter::partitionPath(alnDir, static_cast(bucket))); diff --git a/src/linclust/kmerreduceparallel.cpp b/src/linclust/kmerreduceparallel.cpp index 4cbd59e04..e20cb9c4b 100644 --- a/src/linclust/kmerreduceparallel.cpp +++ b/src/linclust/kmerreduceparallel.cpp @@ -517,6 +517,11 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { // partition are already threaded, and a partition is sized to fill a node. const bool finished = queue.drain(workerId, [&](size_t item) { const size_t partition = waveFrom + item; + // Stamps every block this partition writes with (partition, worker). + // If this worker dies before the queue records the item done, another + // redoes it and the align stage drops these blocks in favour of the + // redo's -- without which both copies would be summed. + edgeWriter->beginPartition(static_cast(partition), workerId); if (info.maxSeqLen < SHRT_MAX) { edgeCount += reducePartition(kmerDir, static_cast(partition), dbType, par, subMat, *edgeWriter, bucketSpan); diff --git a/src/linclust/translatekeys.cpp b/src/linclust/translatekeys.cpp index 9b13046c7..f20b28aea 100644 --- a/src/linclust/translatekeys.cpp +++ b/src/linclust/translatekeys.cpp @@ -62,8 +62,23 @@ struct __attribute__((__packed__)) KeyPair { uint64_t other; // the column carried through untouched }; -// Spill file holding (key, text) records. Needed for the second pass, where the -// column carried through has already become an accession and so varies in width. +// One row after its member has been translated: the representative still a key, +// the member already an accession. The member's key is carried alongside purely +// to order the output (see the sort in pass 3). +struct TranslatedRow { + uint64_t repKey; + uint64_t memberKey; + std::string memberAccession; +}; + +bool byRepThenMember(const TranslatedRow &a, const TranslatedRow &b) { + if (a.repKey != b.repKey) return a.repKey < b.repKey; + return a.memberKey < b.memberKey; +} + +// Spill file holding (key, key, text) records. Needed for the second pass, where +// the column carried through has already become an accession and so varies in +// width. class TextBuckets { public: TextBuckets(const std::string &prefix, unsigned int count, size_t flushBytes = 8 << 20) @@ -73,10 +88,11 @@ class TextBuckets { } ~TextBuckets() { close(); } - void append(unsigned int b, uint64_t key, const char *text, size_t length) { + void append(unsigned int b, uint64_t key, uint64_t other, const char *text, size_t length) { std::string &buf = buffers[b]; const uint32_t len = static_cast(length); buf.append(reinterpret_cast(&key), sizeof(key)); + buf.append(reinterpret_cast(&other), sizeof(other)); buf.append(reinterpret_cast(&len), sizeof(len)); buf.append(text, length); if (buf.size() >= flushBytes) { @@ -97,10 +113,8 @@ class TextBuckets { return prefix + "." + SSTR(b); } - // Reads one bucket back as (key, accession) pairs. - static void read(const std::string &prefix, unsigned int b, - std::vector > &out) { - out.clear(); + // Appends one bucket's rows to out, which the caller accumulates across shards. + static void read(const std::string &prefix, unsigned int b, std::vector &out) { const std::string p = path(prefix, b); if (FileUtil::fileExists(p.c_str()) == false) return; const size_t bytes = FileUtil::getFileSize(p); @@ -113,15 +127,18 @@ class TextBuckets { } fclose(f); size_t at = 0; - while (at + sizeof(uint64_t) + sizeof(uint32_t) <= bytes) { - uint64_t key; + while (at + 2 * sizeof(uint64_t) + sizeof(uint32_t) <= bytes) { + TranslatedRow row; uint32_t len; - memcpy(&key, blob.data() + at, sizeof(key)); - at += sizeof(key); + memcpy(&row.repKey, blob.data() + at, sizeof(row.repKey)); + at += sizeof(row.repKey); + memcpy(&row.memberKey, blob.data() + at, sizeof(row.memberKey)); + at += sizeof(row.memberKey); memcpy(&len, blob.data() + at, sizeof(len)); at += sizeof(len); - out.push_back(std::make_pair(key, blob.substr(at, len))); + row.memberAccession.assign(blob.data() + at, len); at += len; + out.push_back(row); } } @@ -425,7 +442,7 @@ int translatekeys(int argc, const char **argv, const Command &command) { for (size_t i = 0; i < pairs.size(); i++) { const std::string &name = accessionOf(slice, pairs[i].key, lo, lookupFile); writers[tid]->append(static_cast(pairs[i].other / span), - pairs[i].other, name.c_str(), name.size()); + pairs[i].other, pairs[i].key, name.c_str(), name.size()); } FileUtil::remove(KeyBuckets::path(tmpA, b).c_str()); } @@ -438,14 +455,23 @@ int translatekeys(int argc, const char **argv, const Command &command) { // Pass 3: translate the representative and emit. // // Also parallel over buckets, each writing its own piece; the pieces are then - // concatenated in bucket order, so the output is exactly what the sequential - // version produced. + // concatenated in bucket order. + // + // Each bucket's rows are sorted by (representative key, member key) before + // they are written, which is what makes the output reproducible. Pass 2 spills + // to thread-private shards, so which shard a row lands in depends on which + // thread happened to take its source bucket -- and with `schedule(dynamic)` + // that varies between two runs of the same binary on the same input, not just + // between thread counts. Sorting on the keys, which are a property of the data + // alone, removes the schedule from the result entirely. Keys rather than + // accessions because the comparison is then an integer one, and because it + // makes the output order match the key-space clustering this translates. uint64_t written = 0; { #pragma omp parallel num_threads(threads) reduction(+ : written) { std::vector slice; - std::vector > entries; + std::vector entries; std::string buffer; #pragma omp for schedule(dynamic, 1) for (int64_t bi = 0; bi < static_cast(buckets); bi++) { @@ -458,17 +484,9 @@ int translatekeys(int argc, const char **argv, const Command &command) { LookupCursor cursor(lookupFile, lookupAt[b]); cursor.slice(lo, hi, slice); buffer.clear(); + entries.clear(); for (int t = 0; t < threads; t++) { TextBuckets::read(tmpB + ".t" + SSTR(t), b, entries); - for (size_t i = 0; i < entries.size(); i++) { - const std::string &repName = - accessionOf(slice, entries[i].first, lo, lookupFile); - buffer.append(repName); - buffer.push_back('\t'); - buffer.append(entries[i].second); - buffer.push_back('\n'); - written++; - } // A (thread, bucket) shard exists only if that thread wrote // to that bucket, which is sparse. const std::string shard = TextBuckets::path(tmpB + ".t" + SSTR(t), b); @@ -476,6 +494,18 @@ int translatekeys(int argc, const char **argv, const Command &command) { FileUtil::remove(shard.c_str()); } } + // std::sort, not SORT_PARALLEL: this already runs inside the + // parallel region, one bucket per thread. + std::sort(entries.begin(), entries.end(), byRepThenMember); + for (size_t i = 0; i < entries.size(); i++) { + const std::string &repName = + accessionOf(slice, entries[i].repKey, lo, lookupFile); + buffer.append(repName); + buffer.push_back('\t'); + buffer.append(entries[i].memberAccession); + buffer.push_back('\n'); + written++; + } FILE *piece = FileUtil::openAndDelete((outTsv + ".p" + SSTR(b)).c_str(), "w"); writeAllOrDie(buffer.data(), buffer.size(), piece, outTsv); if (fclose(piece) != 0) { diff --git a/src/test/TestParallelCoordination.cpp b/src/test/TestParallelCoordination.cpp index 713a17197..1ac293eb8 100644 --- a/src/test/TestParallelCoordination.cpp +++ b/src/test/TestParallelCoordination.cpp @@ -277,6 +277,36 @@ static void testResumeKeepsProgress(const std::string &dir) { } } +// The reduce names one authoritative producer per item so the align stage can +// drop a crashed worker's superseded output. That rests on complete() keeping the +// *first* worker to record an item, and on those records being readable without +// knowing the item count. +static void testCompletedWorkersNamesOneProducer(const std::string &dir) { + const std::string queuePath = dir + "/authority_queue"; + std::vector workers; + check(WorkQueue::readCompletedWorkers(queuePath + "_missing", workers) == false, + "a queue that does not exist reads as absent"); + + WorkQueue queue(queuePath, 3); + check(queue.claim(7, 60) == 0, "worker 7 takes item 0"); + queue.complete(0, 7); + + // Item 1 is claimed with a lease that lapses, redone by another worker, and + // then also completed by the original -- exactly the race the filter exists + // for. The first worker to record it must stay the authority. + check(queue.claim(8, 1) == 1, "worker 8 takes item 1"); + sleep(2); + check(queue.claim(9, 60) == 1, "the lapsed item is handed to worker 9"); + queue.complete(1, 9); + queue.complete(1, 8); + + check(WorkQueue::readCompletedWorkers(queuePath, workers), "the queue reads back"); + check(workers.size() == 3, "one entry per item"); + check(workers[0] == 7, "item 0 names its producer"); + check(workers[1] == 9, "a redone item names the worker that recorded it first"); + check(workers[2] == -1, "an unfinished item names nobody"); +} + int main(int, const char**) { std::string dir = makeTempDir(); @@ -287,6 +317,7 @@ int main(int, const char**) { testLeaseExpiryRecovers(dir); testReleaseRequeues(dir); testResumeKeepsProgress(dir); + testCompletedWorkersNamesOneProducer(dir); removeTempDir(dir); From bc535566bce8bb63cce4e13579f7a5c97cf5a196 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Mon, 3 Aug 2026 08:38:39 +0000 Subject: [PATCH 11/27] Fix invalid inline documentation. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Björn Buschkämper --- src/linclust/CandidateEdge.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/linclust/CandidateEdge.h b/src/linclust/CandidateEdge.h index a709d130f..44d072611 100644 --- a/src/linclust/CandidateEdge.h +++ b/src/linclust/CandidateEdge.h @@ -57,14 +57,14 @@ struct __attribute__((__packed__)) CandidateEdge { } }; -// Buffered writer for one partition's edge file. +// Buffered writer for one bucket's edge file, used for the *alignment* output: +// alignparallel writes one of these per bucket and greedycluster reads them back. // -// One file per partition, written whole, so a partition redone after a crash -// simply overwrites its file. That makes the reduce idempotent, unlike the map, -// whose per-worker shards can retain a dead worker's partial output. -// -// Used for the reference layout (`--align`, small scale). Production uses -// EdgeBucketWriter below. +// One file per bucket, written under a per-process temporary name and renamed on +// close. The rename is atomic, so a bucket redone after a crash -- or run twice +// because a lease lapsed -- replaces the earlier file with an equally complete +// one rather than appending to it. That is why this stage needs no block framing, +// unlike EdgeBucketWriter below, whose output several producers share. class EdgeWriter { public: EdgeWriter(const std::string &path, size_t bufferRecords = 1024 * 1024); From 3742f78bf988b29533d29c4d6a438c3db765e922 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Mon, 3 Aug 2026 09:19:00 +0000 Subject: [PATCH 12/27] Changed parallel linclust param from mpi-runner to runner. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Björn Buschkämper --- data/workflow/CMakeLists.txt | 1 + data/workflow/linclustparallel.sh | 20 ++++++++ src/CommandDeclarations.h | 1 + src/MMseqsBase.cpp | 19 ++++++++ src/commons/Command.cpp | 3 ++ src/commons/Command.h | 3 ++ src/commons/Parameters.cpp | 28 +++++++++++- src/commons/Parameters.h | 6 +++ src/workflow/CMakeLists.txt | 1 + src/workflow/LinclustParallel.cpp | 76 +++++++++++++++++++++++++++++++ 10 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 src/workflow/LinclustParallel.cpp diff --git a/data/workflow/CMakeLists.txt b/data/workflow/CMakeLists.txt index ac5d29f2c..c2abb0181 100644 --- a/data/workflow/CMakeLists.txt +++ b/data/workflow/CMakeLists.txt @@ -10,6 +10,7 @@ set(GENERATED_WORKFLOWS workflow/map.sh workflow/rbh.sh workflow/linclust.sh + workflow/linclustparallel.sh workflow/clustering.sh workflow/cascaded_clustering.sh workflow/update_clustering.sh diff --git a/data/workflow/linclustparallel.sh b/data/workflow/linclustparallel.sh index e3ffcab74..55592ca3f 100755 --- a/data/workflow/linclustparallel.sh +++ b/data/workflow/linclustparallel.sh @@ -127,6 +127,18 @@ ALIGN_PAR="--min-seq-id $MIN_SEQ_ID --min-aln-len 0 --seq-id-mode 0 -e $EVAL -c # 0. Sequence database with dense, length-ranked keys. Every later stage depends # on that ordering: it is what makes the greedy a single forward sweep. DB="$INPUT" +if [ -f "$INPUT.dbtype" ] && notExists "$INPUT.index.bin"; then + # Checked here rather than left to the first stage. Reaching for `createdb` + # first is the natural thing to do, and its databases are not dense or + # length-ranked, which every stage below assumes. The stage does say so, but + # under a runner that message lands in one worker's log after the run has + # already started; this fails before anything is launched. + fail "$INPUT was not built by createdbparallel: no $INPUT.index.bin. +Every stage addresses sequences as dense, length-ranked keys, which is what makes +the greedy a single forward sweep, and a createdb database has neither. Either +pass the FASTA and let this build the database, or build it with +'mmseqs createdbparallel'." +fi if notExists "$INPUT.dbtype"; then DB="$TMP/db" # Guarded on the finalize sentinel, not on .dbtype: createdbparallel writes @@ -225,4 +237,12 @@ if notExists "$OUT"; then fi fi +# Only on success, and only the whole directory: the per-stage intermediates are +# already dropped as they die (dropIntermediate above), which is what keeps the +# run inside --scratch-budget. This is the stock --remove-tmp-files contract. +if [ -n "$REMOVE_TMP" ]; then + echo "Removing temporary files" + rm -rf "$TMP" +fi + echo "Wrote $OUT" diff --git a/src/CommandDeclarations.h b/src/CommandDeclarations.h index 3f19ae8f8..54fb81463 100644 --- a/src/CommandDeclarations.h +++ b/src/CommandDeclarations.h @@ -84,6 +84,7 @@ extern int lca(int argc, const char **argv, const Command& command); extern int lcaalign(int argc, const char **argv, const Command& command); extern int taxonomyreport(int argc, const char **argv, const Command& command); extern int linclust(int argc, const char **argv, const Command& command); +extern int linclustparallel(int argc, const char **argv, const Command& command); extern int map(int argc, const char **argv, const Command& command); extern int renamedbkeys(int argc, const char **argv, const Command& command); extern int majoritylca(int argc, const char **argv, const Command& command); diff --git a/src/MMseqsBase.cpp b/src/MMseqsBase.cpp index 2b5ae4ec4..0c3f630fc 100644 --- a/src/MMseqsBase.cpp +++ b/src/MMseqsBase.cpp @@ -309,6 +309,25 @@ std::vector baseCommands = { CITATION_MMSEQS2|CITATION_LINCLUST, {{"sequenceDB", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::sequenceDb }, {"clusterDB", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::clusterDb }, {"tmpDir", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::directory }}}, + {"linclustparallel", linclustparallel, &par.linclustparallelworkflow, COMMAND_MAIN, + "Linclust across many nodes over a shared filesystem", + "# Same clustering as linclust, computed by many independent worker\n" + "# processes that coordinate only through files: no MPI, no rank argument,\n" + "# and no node-to-node communication. Every worker of a stage runs the\n" + "# identical command line, so a stage maps onto a Slurm array job and\n" + "# workers may join late, die, or be restarted. Re-running resumes.\n\n" + "# On one node\n" + "mmseqs linclustparallel sequenceDB clusters.tsv tmp\n\n" + "# Across 64 nodes, bounding scratch at 100 TB\n" + "mmseqs linclustparallel sequenceDB clusters.tsv tmp \\\n" + " --runner \"srun -n 64\" --scratch-budget 100T --split-memory-limit 700G\n\n" + "# Output is representativemember in accessions, not a cluster DB:\n" + "# a per-key index is state no single node can hold at this scale.\n", + "Bjoern Buschkaemper ", + " ", + CITATION_MMSEQS2|CITATION_LINCLUST, {{"fastaFile|sequenceDB", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::flatfileAndSequenceDb }, + {"clusterTsv", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::flatfile }, + {"tmpDir", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::directory }}}, {"cluster", clusteringworkflow, &par.clusterworkflow, COMMAND_MAIN, "Slower, sensitive clustering", "# Cascaded clustering of FASTA file\n" diff --git a/src/commons/Command.cpp b/src/commons/Command.cpp index ac6a8b14f..0580f1b65 100644 --- a/src/commons/Command.cpp +++ b/src/commons/Command.cpp @@ -55,6 +55,9 @@ std::vector DbValidator::nuclAaDb = {Parameters::DBTYPE_NUCLEOTIDES, Parame std::vector DbValidator::alignmentDb = {Parameters::DBTYPE_ALIGNMENT_RES}; std::vector DbValidator::directory = {Parameters::DBTYPE_DIRECTORY}; std::vector DbValidator::flatfile = {Parameters::DBTYPE_FLATFILE}; +std::vector DbValidator::flatfileAndSequenceDb = {Parameters::DBTYPE_FLATFILE, Parameters::DBTYPE_INDEX_DB, + Parameters::DBTYPE_NUCLEOTIDES, Parameters::DBTYPE_HMM_PROFILE, + Parameters::DBTYPE_AMINO_ACIDS}; std::vector DbValidator::flatfileAndStdin = {Parameters::DBTYPE_FLATFILE, Parameters::DBTYPE_STDIN}; std::vector DbValidator::flatfileStdinAndGeneric = {Parameters::DBTYPE_FLATFILE, Parameters::DBTYPE_STDIN, Parameters::DBTYPE_GENERIC_DB}; std::vector DbValidator::flatfileStdinGenericUri = {Parameters::DBTYPE_FLATFILE, Parameters::DBTYPE_STDIN, Parameters::DBTYPE_GENERIC_DB, Parameters::DBTYPE_URI}; diff --git a/src/commons/Command.h b/src/commons/Command.h index 07e64ea60..7980cebad 100644 --- a/src/commons/Command.h +++ b/src/commons/Command.h @@ -68,6 +68,9 @@ struct DbValidator { static std::vector taxonomyReportInput; static std::vector directory; static std::vector flatfile; + // For a workflow that builds its own database when handed a FASTA, and + // reuses one when handed a database. + static std::vector flatfileAndSequenceDb; static std::vector flatfileAndStdin; static std::vector flatfileStdinAndGeneric; static std::vector flatfileStdinGenericUri; diff --git a/src/commons/Parameters.cpp b/src/commons/Parameters.cpp index 28286c27c..b175e357b 100644 --- a/src/commons/Parameters.cpp +++ b/src/commons/Parameters.cpp @@ -185,6 +185,7 @@ Parameters::Parameters(): PARAM_CLUSTER_VERSION(PARAM_CLUSTER_VERSION_ID, "--cluster-version", "Cluster version", "Cluster version: 1: Cluster1, 2: Cluster2", typeid(int), (void *) &clusterVersion, "^[1-2]$", MMseqsParameter::COMMAND_CLUSTLINEAR | MMseqsParameter::COMMAND_EXPERT), // workflow PARAM_RUNNER(PARAM_RUNNER_ID, "--mpi-runner", "MPI runner", "Use MPI on compute cluster with this MPI command (e.g. \"mpirun -np 42\")", typeid(std::string), (void *) &runner, "", MMseqsParameter::COMMAND_COMMON | MMseqsParameter::COMMAND_EXPERT), + PARAM_WORKER_RUNNER(PARAM_WORKER_RUNNER_ID, "--runner", "Worker runner", "Command that starts one worker process per node (e.g. \"srun -n 64\"). The workers coordinate through files and never communicate, so this only has to start them; no MPI is involved", typeid(std::string), (void *) &workerRunner, "", MMseqsParameter::COMMAND_COMMON), PARAM_REUSELATEST(PARAM_REUSELATEST_ID, "--force-reuse", "Force restart with latest tmp", "Reuse tmp filse in tmp/latest folder ignoring parameters and version changes", typeid(bool), (void *) &reuseLatest, "", MMseqsParameter::COMMAND_COMMON | MMseqsParameter::COMMAND_EXPERT), // search workflow PARAM_NUM_ITERATIONS(PARAM_NUM_ITERATIONS_ID, "--num-iterations", "Search iterations", "Number of iterative profile search iterations", typeid(int), (void *) &numIterations, "^[1-9]{1}[0-9]*$", MMseqsParameter::COMMAND_PROFILE), @@ -1618,6 +1619,25 @@ Parameters::Parameters(): linclustworkflow.push_back(&PARAM_REUSELATEST); linclustworkflow.push_back(&PARAM_RUNNER); + // linclustparallelworkflow + // Deliberately small. Everything the stages need beyond this is either fixed + // by the algorithm (the k-mer length and alphabet follow from --min-seq-id, + // the extraction settings are the two passes' defaults) or derived from + // --scratch-budget and --split-memory-limit. Exposing the individual stages' + // knobs here would let a run be given parameters its stages disagree on, + // which produces a different clustering rather than an error. + linclustparallelworkflow.push_back(&PARAM_MIN_SEQ_ID); + linclustparallelworkflow.push_back(&PARAM_C); + linclustparallelworkflow.push_back(&PARAM_COV_MODE); + linclustparallelworkflow.push_back(&PARAM_E); + linclustparallelworkflow.push_back(&PARAM_SCRATCH_BUDGET); + linclustparallelworkflow.push_back(&PARAM_SPLIT_MEMORY_LIMIT); + linclustparallelworkflow.push_back(&PARAM_THREADS); + linclustparallelworkflow.push_back(&PARAM_REMOVE_TMP_FILES); + linclustparallelworkflow.push_back(&PARAM_REUSELATEST); + linclustparallelworkflow.push_back(&PARAM_WORKER_RUNNER); + linclustparallelworkflow.push_back(&PARAM_V); + // easylinclustworkflow easylinclustworkflow = combineList(linclustworkflow, createdb); @@ -2719,6 +2739,10 @@ void Parameters::setDefaults() { } else { runner = ""; } + // Not seeded from $RUNNER: the parallel workflow *sets* that variable for the + // driver it execs, so inheriting it here would make a nested invocation + // silently launch its own workers. + workerRunner = ""; reuseLatest = false; // Clustering workflow removeTmpFiles = false; @@ -3110,8 +3134,8 @@ size_t Parameters::hashParameter(const std::vector &dbtypes, const std:: std::string Parameters::createParameterString(const std::vector &par, bool wasSet) { std::ostringstream ss; for (size_t i = 0; i < par.size(); ++i) { - // Never pass the MPI parameters along, they are passed by the environment - if (par[i]->uniqid == PARAM_RUNNER_ID) { + // Never pass the runner parameters along, they are passed by the environment + if (par[i]->uniqid == PARAM_RUNNER_ID || par[i]->uniqid == PARAM_WORKER_RUNNER_ID) { continue; } if(wasSet == true){ diff --git a/src/commons/Parameters.h b/src/commons/Parameters.h index c514f982b..140511709 100644 --- a/src/commons/Parameters.h +++ b/src/commons/Parameters.h @@ -480,6 +480,10 @@ class Parameters { // workflow std::string runner; + // Separate from `runner` on purpose. That one is MPI's, and renaming it would + // break every existing --mpi-runner invocation; this one launches independent + // worker processes that never communicate. + std::string workerRunner; bool reuseLatest; // CLUSTERING @@ -976,6 +980,7 @@ class Parameters { // workflow PARAMETER(PARAM_RUNNER) + PARAMETER(PARAM_WORKER_RUNNER) PARAMETER(PARAM_REUSELATEST) // search workflow @@ -1264,6 +1269,7 @@ class Parameters { std::vector countkmer; std::vector easylinclustworkflow; std::vector linclustworkflow; + std::vector linclustparallelworkflow; std::vector easysearchworkflow; std::vector searchworkflow; std::vector linsearchworkflow; diff --git a/src/workflow/CMakeLists.txt b/src/workflow/CMakeLists.txt index ac5cfcc8e..64352ffc0 100644 --- a/src/workflow/CMakeLists.txt +++ b/src/workflow/CMakeLists.txt @@ -3,6 +3,7 @@ set(workflow_source_files workflow/ClusterUpdate.cpp workflow/Databases.cpp workflow/Linclust.cpp + workflow/LinclustParallel.cpp workflow/EasySearch.cpp workflow/EasyRbh.cpp workflow/EasyCluster.cpp diff --git a/src/workflow/LinclustParallel.cpp b/src/workflow/LinclustParallel.cpp new file mode 100644 index 000000000..7793dc6fe --- /dev/null +++ b/src/workflow/LinclustParallel.cpp @@ -0,0 +1,76 @@ +#include "ByteParser.h" +#include "CommandCaller.h" +#include "Debug.h" +#include "FileUtil.h" +#include "Parameters.h" +#include "Util.h" + +#include "linclustparallel.sh.h" + +#include + +// Shared-filesystem parallel linclust. +// +// Same result as `linclust`, computed by many independent worker processes that +// coordinate only through files. The stages are ordinary MMseqs2 commands run +// with byte-identical arguments on every worker, so a stage maps onto a Slurm +// array job and workers may join late, die, or be restarted: +// +// mmseqs linclustparallel seqDB clusters.tsv tmp --runner "srun -n 64" +// +// The parameter surface is deliberately narrower than `linclust`'s. See the +// comment on linclustparallelworkflow in Parameters.cpp for why, and the header +// of data/workflow/linclustparallel.sh for what the stages are. +// A ByteParser value the stages will parse back. `0` means "unset" to every one +// of them and has no unit; anything else carries an explicit B so it is not read +// as megabytes. +static std::string byteValue(size_t bytes) { + return bytes == 0 ? std::string("0") : ByteParser::format(bytes, 'B', 'l'); +} + +void setLinclustParallelWorkflowDefaults(Parameters *p) { + p->covThr = 0.8; + p->covMode = Parameters::COV_MODE_TARGET; + p->evalThr = 0.001; + p->seqIdThr = 0.9; +} + +int linclustparallel(int argc, const char **argv, const Command &command) { + Parameters &par = Parameters::getInstance(); + setLinclustParallelWorkflowDefaults(&par); + par.parseParameters(argc, argv, command, true, 0, 0); + + std::string tmpDir = par.db3; + std::string hash = + SSTR(par.hashParameter(command.databases, par.filenames, par.linclustparallelworkflow)); + if (par.reuseLatest) { + hash = FileUtil::getHashFromSymLink(tmpDir + "/latest"); + } + tmpDir = FileUtil::createTemporaryDirectory(tmpDir, hash); + par.filenames.pop_back(); + par.filenames.push_back(tmpDir); + + CommandCaller cmd; + // --runner, not --mpi-runner: this only has to start one worker per node, and + // the workers coordinate through files rather than communicating. + cmd.addVariable("RUNNER", par.workerRunner.c_str()); + cmd.addVariable("THREADS", SSTR(par.threads).c_str()); + cmd.addVariable("MIN_SEQ_ID", SSTR(par.seqIdThr).c_str()); + cmd.addVariable("COV", SSTR(par.covThr).c_str()); + cmd.addVariable("COV_MODE", SSTR(par.covMode).c_str()); + cmd.addVariable("EVAL", SSTR(par.evalThr).c_str()); + // Through ByteParser, not as a bare number: the stages parse these with it, + // and it reads an unsuffixed value as *megabytes*, so passing the byte count + // straight through would inflate both limits a millionfold. + cmd.addVariable("SPLIT_MEMORY_LIMIT", byteValue(par.splitMemoryLimit).c_str()); + cmd.addVariable("SCRATCH_BUDGET", byteValue(par.scratchBudget).c_str()); + cmd.addVariable("REMOVE_TMP", par.removeTmpFiles ? "TRUE" : NULL); + + std::string program = tmpDir + "/linclustparallel.sh"; + FileUtil::writeFile(program, linclustparallel_sh, linclustparallel_sh_len); + cmd.execProgram(program.c_str(), par.filenames); + + // Unreachable + assert(false); + return 0; +} From 01d70bfaa039eec64bd8ccac66d967613a4d69b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Mon, 3 Aug 2026 09:36:56 +0000 Subject: [PATCH 13/27] Fix candidate edges header bug. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Björn Buschkämper --- src/linclust/CandidateEdge.cpp | 17 ++++++++++++++-- src/linclust/CandidateEdge.h | 34 +++++++++++++++---------------- src/workflow/LinclustParallel.cpp | 1 + 3 files changed, 33 insertions(+), 19 deletions(-) diff --git a/src/linclust/CandidateEdge.cpp b/src/linclust/CandidateEdge.cpp index 75934a9d0..1d4fe3224 100644 --- a/src/linclust/CandidateEdge.cpp +++ b/src/linclust/CandidateEdge.cpp @@ -158,10 +158,17 @@ void EdgeBucketWriter::flush(unsigned int bucket) { // Header then records, as one write each. The header names the producer so a // crashed worker's superseded copy can be told apart from a second partition // that legitimately produced the same edges (see EdgeBlockHeader). + if (currentWorker < 0) { + // Refused rather than stamped with a placeholder: worker 0 is a real id, + // so an unattributed block would be kept or dropped by coincidence. + Debug(Debug::ERROR) << "Edges were appended to " << dir << " before beginPartition() named " + << "the partition and worker producing them.\n"; + EXIT(EXIT_FAILURE); + } EdgeBlockHeader header; header.magic = EdgeBlockHeader::MAGIC; header.partition = currentPartition; - header.worker = static_cast(currentWorker < 0 ? 0 : currentWorker); + header.worker = static_cast(currentWorker); header.recordCount = static_cast(buffer.size()); if (fwrite(&header, sizeof(EdgeBlockHeader), 1, files[bucket]) != 1) { Debug(Debug::ERROR) << "Cannot write the block header of bucket " << bucket << " of " << dir @@ -254,7 +261,13 @@ size_t EdgeBucketReader::readShard(const std::string &path, const std::vector(header.recordCount) * sizeof(CandidateEdge); diff --git a/src/linclust/CandidateEdge.h b/src/linclust/CandidateEdge.h index 44d072611..f4116c4a1 100644 --- a/src/linclust/CandidateEdge.h +++ b/src/linclust/CandidateEdge.h @@ -93,23 +93,6 @@ class EdgeWriter { }; -// Writes edges into buckets by *representative key range*. -// -// This is the layout the alignment stage needs, and the reason is not obvious: -// aligning inside the k-mer partition fails because a partition's pairs are -// scattered over the whole key space, so every partition ends up re-reading the -// entire sequence database (measured: 52x amplification, see DESIGN_DECISIONS.md -// §9). Bucketing by representative key gives each align worker a contiguous slice -// of sequences instead. -// -// Two things fall out for free: -// - every copy of a pair produced by different k-mer partitions lands in the -// same bucket, so the cross-partition duplicates are removed here rather than -// paid for in duplicate alignments; -// - having all copies together means the per-(pair, diagonal) score -// accumulation stock does in its global merge can be reproduced exactly, -// instead of approximated per partition. -// // Framing for one flushed run of edges, so a record can be traced to the k-mer // partition and the worker that produced it. // @@ -135,6 +118,23 @@ struct __attribute__((__packed__)) EdgeBlockHeader { static const uint32_t MAGIC = 0x45444745; // "EDGE" }; +// Writes edges into buckets by *representative key range*. +// +// This is the layout the alignment stage needs, and the reason is not obvious: +// aligning inside the k-mer partition fails because a partition's pairs are +// scattered over the whole key space, so every partition ends up re-reading the +// entire sequence database (measured: 52x amplification, see DESIGN_DECISIONS.md +// §9). Bucketing by representative key gives each align worker a contiguous slice +// of sequences instead. +// +// Two things fall out for free: +// - every copy of a pair produced by different k-mer partitions lands in the +// same bucket, so the cross-partition duplicates are removed here rather than +// paid for in duplicate alignments; +// - having all copies together means the per-(pair, diagonal) score +// accumulation stock does in its global merge can be reproduced exactly, +// instead of approximated per partition. +// // One file per (worker, bucket), like the k-mer shards, so no locking is needed. class EdgeBucketWriter { public: diff --git a/src/workflow/LinclustParallel.cpp b/src/workflow/LinclustParallel.cpp index 7793dc6fe..4ceca5682 100644 --- a/src/workflow/LinclustParallel.cpp +++ b/src/workflow/LinclustParallel.cpp @@ -21,6 +21,7 @@ // The parameter surface is deliberately narrower than `linclust`'s. See the // comment on linclustparallelworkflow in Parameters.cpp for why, and the header // of data/workflow/linclustparallel.sh for what the stages are. + // A ByteParser value the stages will parse back. `0` means "unset" to every one // of them and has no unit; anything else carries an explicit B so it is not read // as megabytes. From 8cb0a4cdf8b0bf6cb9be11c90f825329c0fdfe91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Mon, 3 Aug 2026 10:49:53 +0000 Subject: [PATCH 14/27] Update in-line comments. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Björn Buschkämper --- src/linclust/CandidateEdge.h | 22 +++++++++++++--------- src/linclust/KmerPartition.h | 13 +++++++------ src/linclust/alignparallel.cpp | 12 +++++++++++- src/linclust/kmerreduceparallel.cpp | 18 ++++++++++-------- 4 files changed, 41 insertions(+), 24 deletions(-) diff --git a/src/linclust/CandidateEdge.h b/src/linclust/CandidateEdge.h index f4116c4a1..eb956157d 100644 --- a/src/linclust/CandidateEdge.h +++ b/src/linclust/CandidateEdge.h @@ -25,15 +25,19 @@ struct __attribute__((__packed__)) CandidateEdge { // How many k-mers put this pair on this diagonal. Stock's prefilter score, // and the value the align stage ranks diagonals by. // - // 16 bits, not 8. The align stage picks a pair's diagonal by comparing these - // counts, so a ceiling low enough to be reached turns a comparison into a tie - // and hands the decision to a tie-break stock does not have. Pass 1 extracts - // 21 k-mers per sequence and never came close; pass 2 uses - // --kmer-per-seq-scale aa:0.100, so a long sequence contributes thousands, and - // at 255 this saturated on 199 of 4.9M records, which gave 23 pairs a - // different diagonal than stock and moved 31 of 1,000,000 sequences into a - // different cluster; widening it here took that residual to 8. The bound is kmersPerSeq(65535) x rounds ~= 26k, which - // fits 16 bits; stock accumulates in an int and has no ceiling at all. + // 16 bits, not 8. At 8 bits this saturated on 199 of 4.9M pass-2 records + // (pass 2 uses --kmer-per-seq-scale aa:0.100, so a long sequence contributes + // thousands), which turned a diagonal comparison into a tie and moved 31 of + // 1,000,000 sequences; widening it took that residual to 8. The bound is + // kmersPerSeq(65535) x rounds ~= 26k, which fits. + // + // Note this is not bit-for-bit stock behaviour, in either width. Stock's + // KmerEntry::score is an unsigned char assigned from an int + // (kmermatcher.h:188, kmermatcher.cpp:2048), so a single entry above 255 + // *wraps* rather than saturating, and the wrapped value is then summed into a + // wider accumulator. Saturating is the more defensible reading of "how many + // k-mers agree on this diagonal" and measured closer to stock, but the two + // can still disagree on a pair whose per-entry count exceeds 255. uint16_t score; uint64_t getRep() const { return get(repBytes); } diff --git a/src/linclust/KmerPartition.h b/src/linclust/KmerPartition.h index 8d7e72104..59f057e42 100644 --- a/src/linclust/KmerPartition.h +++ b/src/linclust/KmerPartition.h @@ -120,14 +120,15 @@ struct KmerShuffleSizing { // memory, P too large and the reduce silently pays P/W sequential re-scans of // the whole database for no benefit. // -// The two scales this is sized for pull in opposite directions and land in -// different places, which is exactly why it should be computed: -// 100B 50.4 TB of k-mers, P = 1024, ~49 GB buckets, ~2 DB scans at W=500 -// 1T 504 TB of k-mers, P = 8192, ~62 GB buckets, ~17 DB scans at W=500 +// The two scales this is sized for pull in opposite directions, which is exactly +// why it should be computed: 50.4 TB of k-mers at 100B against 504 TB at 1T. Note +// that P follows from workerMemoryBytes as much as from the budget, so a target +// scale alone does not pin it. // // persistentBytes is everything sharing the scratch budget with the k-mer wave: -// the sequence database, which persists, plus the surviving edges the fused -// group+align stage accumulates. +// what is already on disk (the caller passes a measured figure, so pass 2 can +// account for what pass 1 left behind) plus the candidate edges this stage's own +// reduce will write. // // scratchBudgetBytes == 0 means unlimited, giving a single wave. KmerShuffleSizing deriveKmerShuffleSizing(uint64_t sequenceCount, diff --git a/src/linclust/alignparallel.cpp b/src/linclust/alignparallel.cpp index 8beceaec5..0d0281d4e 100644 --- a/src/linclust/alignparallel.cpp +++ b/src/linclust/alignparallel.cpp @@ -127,6 +127,17 @@ size_t mergePairCopies(std::vector &edges) { // another worker is a copy left by one that died before recording it. One queue // per extraction wave, and wave w holds the partitions after all earlier waves', // so appending them in wave order indexes by partition directly. +// Which worker's edges count, per k-mer partition. +// +// A partition redone after a crash leaves the dead worker's edges on disk as well +// as the redo's, and mergePairCopies sums matching (pair, diagonal) records, so +// keeping both inflates a diagonal's support. The queue named exactly one +// producer per item -- complete() keeps the first worker to record it -- so +// reading it back says which copy is authoritative. +// +// Waves are concatenated in order because each has its own queue over its own +// contiguous slice of partition space, so wave w's items are partitions +// [w * P / W, (w + 1) * P / W) and appending lands each at its own index. std::vector readReduceAuthority(const std::string &edgeDir) { std::vector authority; for (unsigned int wave = 0;; wave++) { @@ -154,7 +165,6 @@ size_t readBucket(const std::string &edgeDir, unsigned int bucket, return out.size(); } - // The pass-2 acceptance gate stock applies with --filter-cludb-file. // // Before a representative q may take a member t, stock additionally requires that diff --git a/src/linclust/kmerreduceparallel.cpp b/src/linclust/kmerreduceparallel.cpp index e20cb9c4b..0c29c944f 100644 --- a/src/linclust/kmerreduceparallel.cpp +++ b/src/linclust/kmerreduceparallel.cpp @@ -549,14 +549,16 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { // intermediate the pipeline writes. // // Safe here because drain() returns only once every partition of the wave - // is recorded done, which means no worker is still reading one. Several - // workers reach this together, so a file another already unlinked is not - // an error. - // Only the worker that recorded the last completion deletes. drain() - // returns true for every worker that observes the queue finished, and a - // worker whose lease lapsed could still be inside reducePartition; letting - // them all unlink races that reader. Heartbeats make the lapse unlikely, - // this makes the deletion single-writer regardless. + // is recorded done, so no worker holding a live lease is still reading + // one. Every worker that observes the queue finished reaches this and + // unlinks, hence the ENOENT tolerance below. + // + // The one reader this does not account for is a worker whose lease lapsed + // while it was still inside reducePartition: its item was redone and + // recorded by someone else, so the queue reads finished while it is still + // opening shards. Its own edges are already discarded by block header, so + // the outcome is a spurious open failure rather than a wrong answer, and + // the heartbeat makes a lapse unlikely to begin with. for (unsigned int p = waveFrom; p < waveTo; p++) { const std::vector shards = KmerBucketReader::shardFiles(kmerDir, p); for (size_t i = 0; i < shards.size(); i++) { From ac83fe3c0bf8de508f9448160bb51211fd8e5326 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Mon, 3 Aug 2026 11:11:01 +0000 Subject: [PATCH 15/27] Fix name spelling. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Björn Buschkämper --- src/MMseqsBase.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/MMseqsBase.cpp b/src/MMseqsBase.cpp index 0c3f630fc..b150e944b 100644 --- a/src/MMseqsBase.cpp +++ b/src/MMseqsBase.cpp @@ -323,7 +323,7 @@ std::vector baseCommands = { " --runner \"srun -n 64\" --scratch-budget 100T --split-memory-limit 700G\n\n" "# Output is representativemember in accessions, not a cluster DB:\n" "# a per-key index is state no single node can hold at this scale.\n", - "Bjoern Buschkaemper ", + "Björn Buschkämper ", " ", CITATION_MMSEQS2|CITATION_LINCLUST, {{"fastaFile|sequenceDB", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::flatfileAndSequenceDb }, {"clusterTsv", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::flatfile }, From 80aded272fab84009bf0cca53320f53009304987 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Mon, 3 Aug 2026 11:21:17 +0000 Subject: [PATCH 16/27] Fix cicd for greedycluster and kmer partition test. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Björn Buschkämper --- src/linclust/greedycluster.cpp | 5 ++++- src/test/TestKmerPartition.cpp | 11 +++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/linclust/greedycluster.cpp b/src/linclust/greedycluster.cpp index 5a24e4b88..d6744e13b 100644 --- a/src/linclust/greedycluster.cpp +++ b/src/linclust/greedycluster.cpp @@ -56,10 +56,14 @@ #ifdef OPENMP #include +#endif // fwrite that fails loudly. Every caller here is building a final result file that // the workflow renames into place on success; a short write that goes unnoticed // becomes a truncated clustering the next restart treats as finished. +// +// Outside the OPENMP guard: the callers are not, so a -DREQUIRE_OPENMP=0 build +// (the "Old compilers" CI task) would not compile with it inside. static void writeAllOrDie(const void *data, size_t bytes, FILE *file, const std::string &path) { if (bytes == 0) { return; @@ -70,7 +74,6 @@ static void writeAllOrDie(const void *data, size_t bytes, FILE *file, const std: EXIT(EXIT_FAILURE); } } -#endif namespace { diff --git a/src/test/TestKmerPartition.cpp b/src/test/TestKmerPartition.cpp index add7b9d82..cec57e8c0 100644 --- a/src/test/TestKmerPartition.cpp +++ b/src/test/TestKmerPartition.cpp @@ -251,9 +251,16 @@ static void testLosslessRoundTrip(const std::string &dir) { static void testKmerPositionConversion() { typedef KmerPosition Position; + // The widest id that survives *both* the record's 48-bit field and DBKeyType. + // A hardcoded 48-bit constant silently truncates in the default build, where + // MMSEQS_INT64_IDS is 0 and DBKeyType is uint32_t, so this check has to be + // written against the key width the build actually has. + const uint64_t wideId = + std::min(KmerRecord::MAX_ID, static_cast(DB_KEY_INVALID) - 1); + KmerRecord record; record.kmer = 0x0123456789ABCDEFULL; - record.setId(999999999999ULL); + record.setId(wideId); record.pos = 517; record.seqLen = 30000; for (int i = 0; i < 6; i++) { @@ -267,7 +274,7 @@ static void testKmerPositionConversion() { for (int i = 0; i < 6; i++) { adjacencyKept = adjacencyKept && position.getAdjacentSeq(i) == static_cast(i * 3 + 1); } - check(position.kmer == record.kmer && static_cast(position.id) == 999999999999ULL && + check(position.kmer == record.kmer && static_cast(position.id) == wideId && position.pos == 517, "record converts into the KmerPosition assignGroup consumes"); check(position.getSeqLen() == 30000, From 3f1a0aa0dc56d83085b524dae19e0e6bbc0136cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Mon, 3 Aug 2026 12:14:38 +0000 Subject: [PATCH 17/27] Fix resume crashes. Fix smaller bugs. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Björn Buschkämper --- data/workflow/linclustparallel.sh | 148 ++++++++++++++++++--------- src/MMseqsBase.cpp | 25 +++-- src/commons/Command.cpp | 7 +- src/commons/ParallelCoordination.cpp | 71 +++++++++++-- src/commons/ParallelCoordination.h | 6 ++ src/commons/Parameters.cpp | 5 +- src/linclust/CandidateEdge.cpp | 107 ++++++++++++++++--- src/linclust/CandidateEdge.h | 10 +- src/linclust/KmerPartition.cpp | 14 ++- src/linclust/alignparallel.cpp | 11 +- src/linclust/createrepdb.cpp | 20 ++-- src/linclust/kmermatcherparallel.cpp | 113 ++++++++++++-------- src/linclust/kmerreduceparallel.cpp | 26 +++-- src/util/createdbparallel.cpp | 67 ++++++++++-- src/workflow/LinclustParallel.cpp | 30 +++++- 15 files changed, 488 insertions(+), 172 deletions(-) diff --git a/data/workflow/linclustparallel.sh b/data/workflow/linclustparallel.sh index 55592ca3f..83b773c21 100755 --- a/data/workflow/linclustparallel.sh +++ b/data/workflow/linclustparallel.sh @@ -7,7 +7,7 @@ # stage maps onto a Slurm array job and workers may join late, die, or be # restarted. # -# RUNNER="srun -n 64" ./linclustparallel.sh input.fasta clusters.tsv tmp +# WORKER_RUNNER="srun -n 64" ./linclustparallel.sh input.fasta clusters.tsv tmp # # Two things about this script are load-bearing and should not be "simplified": # @@ -34,8 +34,11 @@ # half-written file is never mistaken for a finished one. Redoing one costs # about an hour at 1e11, comfortably inside a 24 h walltime. [ -z "$MMSEQS" ] && MMSEQS=mmseqs -[ -z "$RUNNER" ] && RUNNER="" -[ -z "$THREADS" ] && THREADS=$(nproc 2>/dev/null || echo 8) +# WORKER_RUNNER, not RUNNER: RUNNER already means the MPI runner to ten other +# workflows and to Parameters.cpp, which seeds par.runner from it. +[ -z "$WORKER_RUNNER" ] && WORKER_RUNNER="" +# getconf, not nproc: nproc is GNU-only and absent on macOS/BSD. +[ -z "$THREADS" ] && THREADS=$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 8) [ -z "$MIN_SEQ_ID" ] && MIN_SEQ_ID=0.9 [ -z "$COV" ] && COV=0.8 [ -z "$COV_MODE" ] && COV_MODE=1 @@ -53,13 +56,25 @@ fail() { echo "Error: $1"; exit 1; } # and sizing it as though the filesystem were empty is what made a 1e11 run derive # a single wave and then peak at ~1.9x its ceiling. Measuring beats modelling here # -- it needs no per-stage accounting and stays right when the pipeline changes. +# +# du -sk rather than -sb: -b is GNU-only. Kibibytes are ample precision against a +# budget in terabytes. "$DB"* rather than "$DB" so the index, .index.bin, .lookup +# and headers are counted -- $DB is a prefix, not a path, and at 1e11 the index +# alone is over a terabyte of what this is meant to measure. printf "%.0f" keeps +# awk from rendering large sums in scientific notation, which --scratch-used +# would reject. scratchUsed() { - du -sb "$TMP" "$DB" 2>/dev/null | awk '{s += $1} END {print s + 0 "B"}' + du -sk "$TMP" "$DB"* 2>/dev/null \ + | awk '{s += $1} END {printf "%.0fB\n", s * 1024}' } # Deletes an intermediate whose consumers have all finished. Nothing downstream # reopens these, and they are the bulk of peak scratch: at 100M the candidate # edges alone are 21 GB and the pass-1 alignments 3 GB. +# +# KEEP_INTERMEDIATE=1 suppresses it. Env-only and deliberately not a parameter: +# it exists to inspect a run's intermediates while debugging, and keeping them +# breaks the scratch budget the run was sized against. dropIntermediate() { [ -n "$KEEP_INTERMEDIATE" ] && return 0 rm -rf "$@" @@ -77,7 +92,10 @@ dropIntermediate() { # The alignment does *not* run per wave. Waves partition k-mer space while edges # are bucketed by representative, so every wave's surviving edges land in the same # buckets and the align runs once, afterwards, over their union. -waveCount() { awk '$1 == "waveCount" { print $2 }' "$1/coord/shuffle.info"; } +# `|| true` so a missing or unreadable manifest reaches the check below rather than +# aborting the script under `sh -e` with awk's own error, which is what used to +# happen and made the friendly message unreachable. +waveCount() { awk '$1 == "waveCount" { print $2 }' "$1/coord/shuffle.info" 2>/dev/null || true; } mapReduceWaves() { # $1 sequence DB, $2 k-mer dir, $3 edge dir, $4... extra map arguments @@ -85,11 +103,11 @@ mapReduceWaves() { # shellcheck disable=SC2086 _used=$(scratchUsed) # shellcheck disable=SC2086 - $RUNNER "$MMSEQS" kmermatcherparallel "$_db" "$_kmer" $KMER_COMMON "$@" --kmer-wave 0 \ + $WORKER_RUNNER "$MMSEQS" kmermatcherparallel "$_db" "$_kmer" $KMER_COMMON "$@" --kmer-wave 0 \ --scratch-used "$_used" \ || fail "kmermatcherparallel died" # shellcheck disable=SC2086 - $RUNNER "$MMSEQS" kmerreduceparallel "$_db" "$_kmer" "$_edges" $REDUCE_PAR --kmer-wave 0 \ + $WORKER_RUNNER "$MMSEQS" kmerreduceparallel "$_db" "$_kmer" "$_edges" $REDUCE_PAR --kmer-wave 0 \ || fail "kmerreduceparallel died" _waves=$(waveCount "$_kmer") [ -z "$_waves" ] && fail "no wave count in $_kmer/coord/shuffle.info" @@ -97,20 +115,25 @@ mapReduceWaves() { while [ "$_w" -lt "$_waves" ]; do echo "--- extraction wave $_w of $_waves ---" # shellcheck disable=SC2086 - $RUNNER "$MMSEQS" kmermatcherparallel "$_db" "$_kmer" $KMER_COMMON "$@" --kmer-wave $_w \ + $WORKER_RUNNER "$MMSEQS" kmermatcherparallel "$_db" "$_kmer" $KMER_COMMON "$@" --kmer-wave $_w \ --scratch-used "$_used" \ || fail "kmermatcherparallel (wave $_w) died" # shellcheck disable=SC2086 - $RUNNER "$MMSEQS" kmerreduceparallel "$_db" "$_kmer" "$_edges" $REDUCE_PAR --kmer-wave $_w \ + $WORKER_RUNNER "$MMSEQS" kmerreduceparallel "$_db" "$_kmer" "$_edges" $REDUCE_PAR --kmer-wave $_w \ || fail "kmerreduceparallel (wave $_w) died" _w=$((_w + 1)) done } -[ "$#" -ne 3 ] && { echo "usage: [RUNNER=\"srun -n N\"] $0 "; exit 1; } +[ "$#" -ne 3 ] && { echo "usage: [WORKER_RUNNER=\"srun -n N\"] $0 "; exit 1; } INPUT="$1" OUT="$2" TMP="$3" +# Refused rather than skipped, as linclust.sh:13 does. Guarding the final write on +# `notExists "$OUT"` instead meant a stale file survived a full run and the script +# still reported success, so the user believed they had clustered and had not. +# Resume is keyed on state under $TMP, never on $OUT. +[ -f "$OUT" ] && fail "$OUT exists already" mkdir -p "$TMP" # Shared by both passes. --min-seq-id belongs here because it selects the k-mer @@ -146,29 +169,48 @@ if notExists "$INPUT.dbtype"; then # that looks complete and is not. The sentinel is written last. if notExists "$DB.coord/finalize.done"; then # shellcheck disable=SC2086 - $RUNNER "$MMSEQS" createdbparallel "$INPUT" "$DB" --threads $THREADS \ + $WORKER_RUNNER "$MMSEQS" createdbparallel "$INPUT" "$DB" $VERBOSITY --threads $THREADS \ || fail "createdbparallel died" fi fi +# Checked here, not by alignparallel, which is where it used to surface. That is +# the far end of pass 1, so a nucleotide input paid createdbparallel plus the whole +# map and reduce -- hours and the entire k-mer shuffle on scratch -- before being +# told it was never supported. Stock routes nucleotides to linclust v1, which has +# no v2 counterpart to port. +case "$("$MMSEQS" dbtype "$DB" 2>/dev/null || echo unknown)" in + *Nucleotide*) + fail "$DB holds nucleotide sequences, which this pipeline does not support. +linclust clusters nucleotides with its v1 path (rescorediagonal), and this is the +parallel form of the v2 path, which is protein-only upstream." + ;; +esac + # ---- pass 1, over the whole database -------------------------------------- -mapReduceWaves "$DB" "$TMP/kmer1" "$TMP/edges1" \ - --spaced-kmer-mode 0 --kmer-per-seq-scale aa:0.000,nucl:0.200 +# Guarded as a whole on clu1.tsv. The stages inside are individually resumable, but +# the pass is not re-enterable once its k-mer buckets have been consumed and its +# intermediates dropped: the reduce would then run over an empty shuffle, emit zero +# edges and exit 0, and a greedycluster over that produces a clustering of +# singletons rather than an error. +if notExists "$TMP/clu1.tsv"; then + mapReduceWaves "$DB" "$TMP/kmer1" "$TMP/edges1" \ + --spaced-kmer-mode 0 --kmer-per-seq-scale aa:0.000,nucl:0.200 -# shellcheck disable=SC2086 - $RUNNER "$MMSEQS" alignparallel "$DB" "$TMP/edges1" "$TMP/aln1" $ALIGN_PAR \ + # shellcheck disable=SC2086 + $WORKER_RUNNER "$MMSEQS" alignparallel "$DB" "$TMP/edges1" "$TMP/aln1" $ALIGN_PAR $VERBOSITY \ || fail "alignparallel (pass 1) died" -# Single node from here to the end of the pass: the greedy sweep is sequential by -# necessity, which is what makes it exact. -if notExists "$TMP/clu1.tsv"; then + # Single node from here to the end of the pass: the greedy sweep is sequential + # by necessity, which is what makes it exact. # shellcheck disable=SC2086 - "$MMSEQS" greedycluster "$DB" "$TMP/aln1" "$TMP/clu1.tsv.tmp" --threads $THREADS \ + "$MMSEQS" greedycluster "$DB" "$TMP/aln1" "$TMP/clu1.tsv.tmp" $VERBOSITY --threads $THREADS \ || fail "greedycluster (pass 1) died" mv -f "$TMP/clu1.tsv.tmp" "$TMP/clu1.tsv" fi -# Both are dead once clu1.tsv exists, and together they are the largest thing pass -# 1 leaves behind for pass 2 to be sized around. +# Dead once clu1.tsv exists, and together the largest thing pass 1 leaves behind +# for pass 2 to be sized around. kmer1 is already gone: the reduce unlinks each +# wave's buckets as it consumes them. dropIntermediate "$TMP/edges1" "$TMP/aln1" # ---- pass 2, over the representatives -------------------------------------- @@ -176,25 +218,28 @@ dropIntermediate "$TMP/edges1" "$TMP/aln1" # array offsets. The key map translates the result back below. if notExists "$TMP/rep.keymap"; then # shellcheck disable=SC2086 - "$MMSEQS" createrepdb "$DB" "$TMP/clu1.tsv" "$TMP/rep" --threads $THREADS \ + "$MMSEQS" createrepdb "$DB" "$TMP/clu1.tsv" "$TMP/rep" $VERBOSITY --threads $THREADS \ || fail "createrepdb died" fi -mapReduceWaves "$TMP/rep" "$TMP/kmer2" "$TMP/edges2" \ - --spaced-kmer-mode 1 --kmer-per-seq-scale aa:0.100,nucl:0.100 +# Guarded as a whole, for the same reason as pass 1. +if notExists "$TMP/clu2_sub.tsv"; then + mapReduceWaves "$TMP/rep" "$TMP/kmer2" "$TMP/edges2" \ + --spaced-kmer-mode 1 --kmer-per-seq-scale aa:0.100,nucl:0.100 -# The filter gate: a representative may only take a member if every sequence of -# that member's pass-1 cluster also aligns to it. Needs the original database and -# the key map, since the clustering it consults is in original keys. -# shellcheck disable=SC2086 - $RUNNER "$MMSEQS" alignparallel "$TMP/rep" "$TMP/edges2" "$TMP/aln2" $ALIGN_PAR \ + # The filter gate: a representative may only take a member if every sequence of + # that member's pass-1 cluster also aligns to it. Needs the original database + # and the key map, since the clustering it consults is in original keys. + # shellcheck disable=SC2086 + $WORKER_RUNNER "$MMSEQS" alignparallel "$TMP/rep" "$TMP/edges2" "$TMP/aln2" $ALIGN_PAR \ + $VERBOSITY \ --filter-cludb-file "$TMP/clu1.tsv" --filter-seqdb-file "$DB" \ --key-map "$TMP/rep.keymap" \ || fail "alignparallel (pass 2) died" -if notExists "$TMP/clu2_sub.tsv"; then # shellcheck disable=SC2086 - "$MMSEQS" greedycluster "$TMP/rep" "$TMP/aln2" "$TMP/clu2_sub.tsv.tmp" --threads $THREADS \ + "$MMSEQS" greedycluster "$TMP/rep" "$TMP/aln2" "$TMP/clu2_sub.tsv.tmp" $VERBOSITY \ + --threads $THREADS \ || fail "greedycluster (pass 2) died" mv -f "$TMP/clu2_sub.tsv.tmp" "$TMP/clu2_sub.tsv" fi @@ -203,7 +248,7 @@ dropIntermediate "$TMP/edges2" "$TMP/aln2" "$TMP/kmer2" if notExists "$TMP/clu2.tsv"; then # shellcheck disable=SC2086 "$MMSEQS" translatecluster "$TMP/clu2_sub.tsv" "$TMP/rep.keymap" "$TMP/clu2.tsv.tmp" \ - --threads $THREADS --split-memory-limit $SPLIT_MEMORY_LIMIT \ + $VERBOSITY --split-memory-limit $SPLIT_MEMORY_LIMIT \ || fail "translatecluster died" mv -f "$TMP/clu2.tsv.tmp" "$TMP/clu2.tsv" fi @@ -214,7 +259,7 @@ fi if notExists "$TMP/clu.keys.tsv"; then # shellcheck disable=SC2086 "$MMSEQS" mergeclusterparallel "$DB" "$TMP/clu1.tsv" "$TMP/clu2.tsv" "$TMP/clu.keys.tsv.tmp" \ - --threads $THREADS --split-memory-limit $SPLIT_MEMORY_LIMIT \ + $VERBOSITY --split-memory-limit $SPLIT_MEMORY_LIMIT \ || fail "mergeclusterparallel died" mv -f "$TMP/clu.keys.tsv.tmp" "$TMP/clu.keys.tsv" fi @@ -224,25 +269,32 @@ fi # id->name table. Here it is a streaming join against the .lookup. Skipped when # the database has none (--write-lookup 0), leaving the key-space result as the # output rather than failing at the last step. -if notExists "$OUT"; then - if [ -f "$DB.lookup" ]; then - # shellcheck disable=SC2086 - "$MMSEQS" translatekeys "$TMP/clu.keys.tsv" "$DB.lookup" "$OUT.tmp" \ - --threads $THREADS --split-memory-limit $SPLIT_MEMORY_LIMIT \ - || fail "translatekeys died" - mv -f "$OUT.tmp" "$OUT" - else - echo "No $DB.lookup; leaving the result in database keys" - cp "$TMP/clu.keys.tsv" "$OUT" - fi +if [ -f "$DB.lookup" ]; then + # shellcheck disable=SC2086 + "$MMSEQS" translatekeys "$TMP/clu.keys.tsv" "$DB.lookup" "$OUT.tmp" $VERBOSITY \ + --threads $THREADS --split-memory-limit $SPLIT_MEMORY_LIMIT \ + || fail "translatekeys died" + mv -f "$OUT.tmp" "$OUT" +else + echo "No $DB.lookup; leaving the result in database keys" + # Copied to a temporary name and renamed, like every other output here: an + # interrupted cp straight onto $OUT leaves a truncated file behind. + cp "$TMP/clu.keys.tsv" "$OUT.tmp" || fail "cannot copy the key-space result" + mv -f "$OUT.tmp" "$OUT" fi -# Only on success, and only the whole directory: the per-stage intermediates are -# already dropped as they die (dropIntermediate above), which is what keeps the -# run inside --scratch-budget. This is the stock --remove-tmp-files contract. +# Only on success, and only the run's own intermediates. The hashed directory +# itself and its `latest` symlink are left in place, as every other workflow does, +# so --force-reuse still resolves; `rm -rf "$TMP"` used to leave `latest` dangling. +# The per-stage intermediates are already dropped as they die (dropIntermediate +# above), which is what keeps the run inside --scratch-budget. if [ -n "$REMOVE_TMP" ]; then echo "Removing temporary files" - rm -rf "$TMP" + rm -rf "$TMP/kmer1" "$TMP/kmer2" "$TMP/edges1" "$TMP/edges2" "$TMP/aln1" "$TMP/aln2" + rm -f "$TMP/clu1.tsv" "$TMP/clu2.tsv" "$TMP/clu2_sub.tsv" "$TMP/clu.keys.tsv" + rm -f "$TMP/rep" "$TMP/rep".* "$TMP/rep_h" "$TMP/rep_h".* + rm -rf "$TMP/db.coord" + rm -f "$TMP/db" "$TMP/db".* "$TMP/db_h" "$TMP/db_h".* fi echo "Wrote $OUT" diff --git a/src/MMseqsBase.cpp b/src/MMseqsBase.cpp index b150e944b..35e7b44f1 100644 --- a/src/MMseqsBase.cpp +++ b/src/MMseqsBase.cpp @@ -168,7 +168,7 @@ std::vector baseCommands = { "mmseqs createdbparallel seq.fasta sequenceDB\n\n" "# Run it from several nodes against the same shared filesystem\n" "srun -N 8 mmseqs createdbparallel seq.fasta sequenceDB --chunk-size 1G\n", - "Martin Steinegger ", + "Björn Buschkämper ", " ... ", CITATION_MMSEQS2, {{"fastaFile", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA | DbType::VARIADIC, &DbValidator::flatfile }, {"sequenceDB", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::flatfile }}}, @@ -706,7 +706,7 @@ std::vector baseCommands = { "mmseqs kmermatcherparallel sequenceDB kmerDir\n\n" "# Run it from several nodes against the same shared filesystem\n" "srun -N 8 mmseqs kmermatcherparallel sequenceDB kmerDir --scratch-budget 100T\n", - "Martin Steinegger ", + "Björn Buschkämper ", " ", CITATION_MMSEQS2, {{"sequenceDB", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::sequenceDb }, {"kmerDir", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::directory }}}, @@ -718,7 +718,7 @@ std::vector baseCommands = { "mmseqs kmerreduceparallel sequenceDB kmerDir edgeDir\n\n" "# Run it from several nodes against the same shared filesystem\n" "srun -N 8 mmseqs kmerreduceparallel sequenceDB kmerDir edgeDir\n", - "Martin Steinegger ", + "Björn Buschkämper ", " ", CITATION_MMSEQS2, {{"sequenceDB", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::sequenceDb }, {"kmerDir", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::directory }, @@ -728,7 +728,7 @@ std::vector baseCommands = { "# Merge the duplicate copies the k-mer partitions produced, align each pair\n" "# once, and write the survivors. Workers claim representative-key buckets.\n" "mmseqs alignparallel sequenceDB edgeDir alnDir --min-seq-id 0.9 -c 0.8\n", - "Martin Steinegger ", + "Björn Buschkämper ", " ", CITATION_MMSEQS2, {{"sequenceDB", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::sequenceDb }, {"edgeDir", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::directory }, @@ -739,7 +739,7 @@ std::vector baseCommands = { "# left-to-right sweep is the exact greedy. Needs two bits per key, not the\n" "# eight bytes per sequence stock's fused clustering keeps resident.\n" "mmseqs greedycluster sequenceDB alnDir clusters.tsv\n", - "Martin Steinegger ", + "Björn Buschkämper ", " ", CITATION_MMSEQS2, {{"sequenceDB", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::sequenceDb }, {"alnDir", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::directory }, @@ -750,19 +750,22 @@ std::vector baseCommands = { "# per-sequence list array is needed (stock keeps 24 B/sequence of empty\n" "# list headers before storing a single member).\n" "mmseqs mergeclusterparallel sequenceDB pass1.tsv pass2.tsv clusters.tsv\n", - "Martin Steinegger ", + "Björn Buschkämper ", " ... ", CITATION_MMSEQS2, {{"sequenceDB", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::sequenceDb }, {"clusterTsv", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA|DbType::VARIADIC, &DbValidator::flatfile }, {"clusterTsv", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::flatfile }}}, - {"createrepdb", createrepdb, &par.createrepdb, COMMAND_DATABASE_CREATION, + {"createrepdb", createrepdb, &par.createrepdb, COMMAND_HIDDEN, "Build a densely re-keyed representative DB for the next linclust pass", + "# An internal stage of linclustparallel, not a general database builder: the\n" + "# output carries a dense companion index instead of the usual .index/.lookup,\n" + "# so only the parallel linclust stages can read it.\n" "# Representatives keep length order, so sub-key i is the i-th representative\n" "# and the copy is sequential. Writes .keymap (sub-key -> original key)\n" "# so the next pass's clustering can be translated back before merging.\n" "mmseqs createrepdb sequenceDB clusters.tsv repDB\n", - "Martin Steinegger ", - " ", + "Björn Buschkämper ", + " ", CITATION_MMSEQS2, {{"sequenceDB", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::sequenceDb }, {"clusterTsv", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::flatfile }, {"sequenceDB", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::flatfile }}}, @@ -772,7 +775,7 @@ std::vector baseCommands = { "# back before merging. Each key column is translated in its own bucketed\n" "# pass, so the key map is only ever read in contiguous slices.\n" "mmseqs translatecluster pass2.tsv repDB.keymap pass2_orig.tsv\n", - "Martin Steinegger ", + "Björn Buschkämper ", " ", CITATION_MMSEQS2, {{"clusterTsv", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::flatfile }, {"keyMap", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::flatfile }, @@ -784,7 +787,7 @@ std::vector baseCommands = { "# key column is translated in its own bucketed pass, and the lookup is read\n" "# sequentially, so nothing per-key is ever resident.\n" "mmseqs translatekeys clusters.tsv sequenceDB.lookup clusters_named.tsv\n", - "Martin Steinegger ", + "Björn Buschkämper ", " ", CITATION_MMSEQS2, {{"clusterTsv", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::flatfile }, {"lookupFile", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::flatfile }, diff --git a/src/commons/Command.cpp b/src/commons/Command.cpp index 0580f1b65..941042753 100644 --- a/src/commons/Command.cpp +++ b/src/commons/Command.cpp @@ -55,8 +55,11 @@ std::vector DbValidator::nuclAaDb = {Parameters::DBTYPE_NUCLEOTIDES, Parame std::vector DbValidator::alignmentDb = {Parameters::DBTYPE_ALIGNMENT_RES}; std::vector DbValidator::directory = {Parameters::DBTYPE_DIRECTORY}; std::vector DbValidator::flatfile = {Parameters::DBTYPE_FLATFILE}; -std::vector DbValidator::flatfileAndSequenceDb = {Parameters::DBTYPE_FLATFILE, Parameters::DBTYPE_INDEX_DB, - Parameters::DBTYPE_NUCLEOTIDES, Parameters::DBTYPE_HMM_PROFILE, +// A FASTA to build from, or an amino-acid database already built. Deliberately not +// index or profile databases, which the parallel linclust stages cannot read, and +// not nucleotides, which its align stage rejects -- advertising those only moves +// the failure later into the run. +std::vector DbValidator::flatfileAndSequenceDb = {Parameters::DBTYPE_FLATFILE, Parameters::DBTYPE_AMINO_ACIDS}; std::vector DbValidator::flatfileAndStdin = {Parameters::DBTYPE_FLATFILE, Parameters::DBTYPE_STDIN}; std::vector DbValidator::flatfileStdinAndGeneric = {Parameters::DBTYPE_FLATFILE, Parameters::DBTYPE_STDIN, Parameters::DBTYPE_GENERIC_DB}; diff --git a/src/commons/ParallelCoordination.cpp b/src/commons/ParallelCoordination.cpp index 4bedf8684..5ee9bbd27 100644 --- a/src/commons/ParallelCoordination.cpp +++ b/src/commons/ParallelCoordination.cpp @@ -195,21 +195,42 @@ void WorkQueue::initialiseLocked() { << ". Remove the coordination directory to start over.\n"; EXIT(EXIT_FAILURE); } + // The records, not the counter, are the truth. completeLocked marks an item + // DONE and then increments doneCount as two separate writes, so a node lost + // between them leaves the count one short -- and since nothing is then + // claimable and allDone() never becomes true, drain() would report the stage + // stalled on every restart, permanently. Recounting once per open costs a + // single bulk read and makes that unreachable. + std::vector records; + readRecordsLocked(records); + uint64_t done = 0; + for (size_t i = 0; i < records.size(); i++) { + if (records[i].state == DONE) { + done++; + } + } + if (done != header.doneCount) { + Debug(Debug::WARNING) << "Work queue " << path << " recorded " << header.doneCount + << " completed items but holds " << done + << "; repairing the count from the records.\n"; + header.doneCount = done; + writeHeaderLocked(header); + fsync(lock.getFd()); + } return; } - Header fresh; - memset(&fresh, 0, sizeof(fresh)); - fresh.magic = MAGIC; - fresh.version = VERSION; - fresh.itemCount = static_cast(itemCount); - fresh.doneCount = 0; - fresh.nextHint = 0; - writeHeaderLocked(fresh); - // Zero-fill the record array so every later access is a plain offset read // rather than a short read off the end of a sparse file. PENDING is state 0, // so zeroing is also the correct initial state. + // + // Records first, header last. The header is what makes the file look like a + // queue, so writing it first meant a death during the zero-fill left a file + // that passed the magic and itemCount checks above over a record array that + // was still short -- every later claim() then read past EOF and exited, for + // every worker, forever, with no recovery but deleting the directory by hand. + // The window is one pwrite for the linclust queues but ~100 for + // createdbparallel at 1e11. const size_t batchSize = 65536; Record *blank = new Record[batchSize]; memset(blank, 0, batchSize * sizeof(Record)); @@ -224,6 +245,16 @@ void WorkQueue::initialiseLocked() { } delete[] blank; fsync(lock.getFd()); + + Header fresh; + memset(&fresh, 0, sizeof(fresh)); + fresh.magic = MAGIC; + fresh.version = VERSION; + fresh.itemCount = static_cast(itemCount); + fresh.doneCount = 0; + fresh.nextHint = 0; + writeHeaderLocked(fresh); + fsync(lock.getFd()); } WorkQueue::Header WorkQueue::readHeaderLocked() { @@ -253,6 +284,19 @@ WorkQueue::Record WorkQueue::readRecordLocked(int64_t index) { return record; } +void WorkQueue::readRecordsLocked(std::vector &out) { + out.resize(static_cast(itemCount)); + if (itemCount == 0) { + return; + } + const size_t bytes = static_cast(itemCount) * sizeof(Record); + if (preadFully(lock.getFd(), &out[0], bytes, recordOffset(0)) != static_cast(bytes)) { + Debug(Debug::ERROR) << "Could not read the " << itemCount << " records of work queue " + << path << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } +} + void WorkQueue::writeRecordLocked(int64_t index, const Record &record) { if (pwriteFully(lock.getFd(), &record, sizeof(record), recordOffset(index)) < 0) { Debug(Debug::ERROR) << "Could not write work queue record " << index << " in " << path @@ -267,9 +311,12 @@ int64_t WorkQueue::claim(int64_t workerId, int64_t leaseSeconds) { Header header = readHeaderLocked(); const int64_t now = nowSeconds(); + std::vector records; + readRecordsLocked(records); + int64_t firstUnfinished = -1; for (int64_t index = static_cast(header.nextHint); index < itemCount; index++) { - Record record = readRecordLocked(index); + Record record = records[static_cast(index)]; if (record.state == DONE) { continue; } @@ -375,9 +422,11 @@ int64_t WorkQueue::getDoneCount() { bool WorkQueue::hasLiveClaim() { const int64_t now = nowSeconds(); lock.lock(); + std::vector records; + readRecordsLocked(records); bool live = false; for (int64_t i = 0; i < itemCount && live == false; i++) { - const Record record = readRecordLocked(i); + const Record &record = records[static_cast(i)]; if (record.state == CLAIMED && static_cast(record.leaseExpiry) > now) { live = true; } diff --git a/src/commons/ParallelCoordination.h b/src/commons/ParallelCoordination.h index 281a48c0b..0aa067d43 100644 --- a/src/commons/ParallelCoordination.h +++ b/src/commons/ParallelCoordination.h @@ -258,6 +258,12 @@ class WorkQueue { Header readHeaderLocked(); void writeHeaderLocked(const Header &header); Record readRecordLocked(int64_t index); + // The whole record array in one read, for the scans that would otherwise issue + // itemCount separate 16-byte preads while holding the lock. On a shared + // filesystem each of those is a cross-node round trip, and at P = 8192 with + // every idle worker polling every 5 s they crowd out the heartbeat renewals + // that keep live workers' leases from lapsing. + void readRecordsLocked(std::vector &out); void writeRecordLocked(int64_t index, const Record &record); void completeLocked(int64_t index, int64_t workerId); diff --git a/src/commons/Parameters.cpp b/src/commons/Parameters.cpp index b175e357b..d6f32fcc2 100644 --- a/src/commons/Parameters.cpp +++ b/src/commons/Parameters.cpp @@ -61,7 +61,7 @@ Parameters::Parameters(): PARAM_KMER_WAVE(PARAM_KMER_WAVE_ID, "--kmer-wave", "K-mer wave", "Which extraction wave to write, when the scratch budget needs more than one. Each wave re-extracts every k-mer but keeps only its own slice of the partition space, so peak scratch is the whole shuffle divided by the wave count. Default -1 requires a single wave. Run waves 0..W-1, reducing each before starting the next.", typeid(int), (void *) &kmerWave, "^-?[0-9]+$", MMseqsParameter::COMMAND_EXPERT), PARAM_KEY_MAP(PARAM_KEY_MAP_ID, "--key-map", "Sub-key map", "Maps this database's dense sub-keys back to the original keys, as written by createrepdb. Needed with --filter-cludb-file when the pass runs on a re-keyed representative database.", typeid(std::string), (void *) &keyMapFile, "", MMseqsParameter::COMMAND_ALIGN | MMseqsParameter::COMMAND_EXPERT), PARAM_SCRATCH_BUDGET(PARAM_SCRATCH_BUDGET_ID, "--scratch-budget", "Scratch budget", "Total scratch the run may occupy. The k-mer extraction wave count and the partition count are derived from this together with --split-memory-limit, rather than set by hand. Default (0) for a single wave. E.g. 100T, 500T", typeid(ByteParser), (void *) &scratchBudget, "^(0|[1-9]{1}[0-9]*(B|K|M|G|T)?)$", MMseqsParameter::COMMAND_EXPERT), - PARAM_SCRATCH_USED(PARAM_SCRATCH_USED_ID, "--scratch-used", "Scratch already used", "Bytes of --scratch-budget already occupied when this stage starts. The workflow measures it, so a later pass accounts for what earlier passes left on disk. 0 derives it from the input database alone.", typeid(ByteParser), (void *) &scratchUsed, "^([1-9]{1}[0-9]*(B|K|M|G|T)?)|0$", MMseqsParameter::COMMAND_EXPERT), + PARAM_SCRATCH_USED(PARAM_SCRATCH_USED_ID, "--scratch-used", "Scratch already used", "Bytes of --scratch-budget already occupied when this stage starts. The workflow measures it, so a later pass accounts for what earlier passes left on disk. 0 derives it from the input database alone.", typeid(ByteParser), (void *) &scratchUsed, "^(0|[1-9]{1}[0-9]*(B|K|M|G|T)?)$", MMseqsParameter::COMMAND_EXPERT), PARAM_DISK_SPACE_LIMIT(PARAM_DISK_SPACE_LIMIT_ID, "--disk-space-limit", "Disk space limit", "Set max disk space to use for reverse profile searches. E.g. 800B, 5K, 10M, 1G. Default (0) to all available disk space in the temp folder", typeid(ByteParser), (void *) &diskSpaceLimit, "^(0|[1-9]{1}[0-9]*(B|K|M|G|T)?)$", MMseqsParameter::COMMAND_COMMON | MMseqsParameter::COMMAND_PREFILTER | MMseqsParameter::COMMAND_EXPERT), PARAM_SPLIT_AMINOACID(PARAM_SPLIT_AMINOACID_ID, "--split-aa", "Split by amino acid", "Try to find the best split boundaries by entry lengths", typeid(bool), (void *) &splitAA, "$", MMseqsParameter::COMMAND_EXPERT), PARAM_SUB_MAT(PARAM_SUB_MAT_ID, "--sub-mat", "Substitution matrix", "Substitution matrix file", typeid(MultiParam>), (void *) &scoringMatrixFile, "", MMseqsParameter::COMMAND_COMMON | MMseqsParameter::COMMAND_EXPERT), @@ -998,17 +998,14 @@ Parameters::Parameters(): // mergeclusterparallel mergeclusterparallel.push_back(&PARAM_SPLIT_MEMORY_LIMIT); - mergeclusterparallel.push_back(&PARAM_THREADS); mergeclusterparallel.push_back(&PARAM_V); // createrepdb createrepdb.push_back(&PARAM_THREADS); - createrepdb.push_back(&PARAM_COMPRESSED); createrepdb.push_back(&PARAM_V); // translatecluster translatecluster.push_back(&PARAM_SPLIT_MEMORY_LIMIT); - translatecluster.push_back(&PARAM_THREADS); translatecluster.push_back(&PARAM_V); // translatekeys diff --git a/src/linclust/CandidateEdge.cpp b/src/linclust/CandidateEdge.cpp index 1d4fe3224..fba2f8700 100644 --- a/src/linclust/CandidateEdge.cpp +++ b/src/linclust/CandidateEdge.cpp @@ -11,14 +11,40 @@ #include #include +#include #include +namespace { + +// Makes a rename durable. Renaming into place is atomic against other readers but +// not against losing the node: the directory entry lives in the parent, so without +// this the file can come back missing or empty after a crash the work queue has +// already recorded as a completed item. +void syncParentDirectory(const std::string &path) { + const size_t slash = path.find_last_of('/'); + const std::string dir = slash == std::string::npos ? std::string(".") : path.substr(0, slash); + const int fd = open(dir.c_str(), O_RDONLY); + if (fd < 0) { + Debug(Debug::ERROR) << "Cannot open " << dir << " to flush it: " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + if (fsync(fd) != 0) { + Debug(Debug::ERROR) << "Cannot flush " << dir << ": " << strerror(errno) << "\n"; + close(fd); + EXIT(EXIT_FAILURE); + } + close(fd); +} + +} // namespace + std::string EdgeWriter::partitionPath(const std::string &dir, unsigned int partition) { return dir + "/p" + SSTR(partition) + ".edges"; } -EdgeWriter::EdgeWriter(const std::string &path, size_t bufferRecords) - : path(path), file(NULL), bufferRecords(bufferRecords), edgeCount(0), closed(false) { +EdgeWriter::EdgeWriter(const std::string &path, int64_t workerId, size_t bufferRecords) + : path(path), workerId(workerId), file(NULL), bufferRecords(bufferRecords), edgeCount(0), + closed(false) { buffer.reserve(bufferRecords); } @@ -38,7 +64,14 @@ void EdgeWriter::flush() { // whole number of records and pass every downstream integrity check. // rename(2) is atomic, so the loser simply replaces the winner with an // equally complete file. - tmpPath = path + ".w" + SSTR(getpid()); + // + // The worker id, not getpid(): pids are unique per node, and this path is + // on a shared filesystem. Two workers on two nodes drawing the same pid -- + // routine across a homogeneous allocation -- would open the same "wb" path + // and truncate each other, publishing a file with a hole of zero-filled + // edges that every downstream size check still accepts. The bucket writers + // alongside this one already key their shards on the worker id. + tmpPath = path + ".w" + SSTR(workerId); file = fopen(tmpPath.c_str(), "wb"); if (file == NULL) { Debug(Debug::ERROR) << "Cannot open edge file " << tmpPath << ": " << strerror(errno) @@ -74,6 +107,18 @@ void EdgeWriter::close() { // A partition that produced no edge still gets an empty file, so a reader // can tell "this partition was reduced and had nothing" from "this // partition was never reduced". + // + // fsync before the rename, and the directory after it, because the work + // queue that vouches for this file *is* fsynced when the item is completed. + // Without this the durability runs the wrong way round: a node lost after + // the rename can replay the metadata without the data extents, leaving a + // zero-filled bucket that the queue records as DONE, so nobody redoes it + // and greedycluster reads it as "no edges" in silence. + if (fflush(file) != 0 || fsync(fileno(file)) != 0) { + Debug(Debug::ERROR) << "Cannot flush edge file " << tmpPath << ": " << strerror(errno) + << "\n"; + EXIT(EXIT_FAILURE); + } if (fclose(file) != 0) { Debug(Debug::ERROR) << "Cannot close edge file " << tmpPath << ": " << strerror(errno) << "\n"; @@ -85,8 +130,9 @@ void EdgeWriter::close() { << strerror(errno) << "\n"; EXIT(EXIT_FAILURE); } + syncParentDirectory(path); } else { - const std::string tmp = path + ".w" + SSTR(getpid()); + const std::string tmp = path + ".w" + SSTR(workerId); FILE *empty = fopen(tmp.c_str(), "wb"); if (empty == NULL) { Debug(Debug::ERROR) << "Cannot create edge file " << tmp << ": " << strerror(errno) @@ -98,6 +144,7 @@ void EdgeWriter::close() { << strerror(errno) << "\n"; EXIT(EXIT_FAILURE); } + syncParentDirectory(path); } } @@ -131,12 +178,17 @@ EdgeBucketWriter::EdgeBucketWriter(const std::string &dir, unsigned int bucketCo edgesPerBuffer = std::max(perBucket, 64); buffers.resize(bucketCount); files.assign(bucketCount, NULL); + written.assign(bucketCount, false); } EdgeBucketWriter::~EdgeBucketWriter() { close(); } +std::string EdgeBucketWriter::shardPath(unsigned int bucket) const { + return bucketDir(dir, bucket) + "/" + shardId + ".edges"; +} + void EdgeBucketWriter::flush(unsigned int bucket) { std::vector &buffer = buffers[bucket]; if (buffer.empty()) { @@ -145,7 +197,7 @@ void EdgeBucketWriter::flush(unsigned int bucket) { if (files[bucket] == NULL) { // Opened lazily: a worker whose partitions produced nothing for a bucket // should not cost a descriptor or an empty file. - const std::string path = bucketDir(dir, bucket) + "/" + shardId + ".edges"; + const std::string path = shardPath(bucket); // Append-and-close per flush, for the same reason as KmerBucketWriter: // one descriptor per bucket would need up to 65536 of them. files[bucket] = fopen(path.c_str(), "ab"); @@ -180,6 +232,17 @@ void EdgeBucketWriter::flush(unsigned int bucket) { << " of " << dir << ": " << strerror(errno) << "\n"; EXIT(EXIT_FAILURE); } + // Closed here, not held until close(): with up to 65536 buckets against the + // 8192 descriptors fixRlimitNoFile raises to, keeping one open per bucket runs + // the process out of descriptors partway through the reduce. This is what the + // comment above the open has always claimed; only the fclose was missing. + if (fclose(files[bucket]) != 0) { + Debug(Debug::ERROR) << "Cannot close edge bucket " << bucket << " of " << dir << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + files[bucket] = NULL; + written[bucket] = true; buffer.clear(); } @@ -206,10 +269,33 @@ void EdgeBucketWriter::append(unsigned int bucket, const CandidateEdge &edge) { void EdgeBucketWriter::flushAll() { for (unsigned int b = 0; b < bucketCount; b++) { flush(b); - if (files[b] != NULL && fflush(files[b]) != 0) { - Debug(Debug::ERROR) << "Cannot flush edge bucket " << b << ": " << strerror(errno) << "\n"; + } + // Then make what was written durable, before the caller marks the work item + // done. The queue fsyncs its own completion record, so without this the record + // that an item finished outlives the data it vouches for: a node lost here + // brings the shard back short or zero-filled, and because the item reads DONE + // nobody redoes it. + // + // Once per item rather than once per flush -- a flush is a few hundred KB at + // large bucket counts, and syncing each would dominate the stage. + for (unsigned int b = 0; b < bucketCount; b++) { + if (written[b] == false) { + continue; + } + const std::string path = shardPath(b); + const int fd = ::open(path.c_str(), O_WRONLY | O_APPEND); + if (fd < 0 || fsync(fd) != 0) { + Debug(Debug::ERROR) << "Cannot flush edge bucket " << path << " to disk: " + << strerror(errno) << "\n"; + if (fd >= 0) { + ::close(fd); + } EXIT(EXIT_FAILURE); } + ::close(fd); + // The shard's own directory, because a shard created during this item is + // only reachable once its directory entry is durable too. + syncParentDirectory(path); } } @@ -220,13 +306,6 @@ void EdgeBucketWriter::close() { closed = true; for (unsigned int b = 0; b < bucketCount; b++) { flush(b); - if (files[b] != NULL) { - if (fclose(files[b]) != 0) { - Debug(Debug::ERROR) << "Cannot close edge bucket " << b << "\n"; - EXIT(EXIT_FAILURE); - } - files[b] = NULL; - } } } diff --git a/src/linclust/CandidateEdge.h b/src/linclust/CandidateEdge.h index eb956157d..006643851 100644 --- a/src/linclust/CandidateEdge.h +++ b/src/linclust/CandidateEdge.h @@ -71,7 +71,10 @@ struct __attribute__((__packed__)) CandidateEdge { // unlike EdgeBucketWriter below, whose output several producers share. class EdgeWriter { public: - EdgeWriter(const std::string &path, size_t bufferRecords = 1024 * 1024); + // workerId names the temporary file this writes before renaming. It has to be + // the globally unique worker id rather than a pid, because the path is on the + // shared filesystem and pids collide across nodes. + EdgeWriter(const std::string &path, int64_t workerId, size_t bufferRecords = 1024 * 1024); ~EdgeWriter(); void append(const CandidateEdge &edge); @@ -88,6 +91,7 @@ class EdgeWriter { void flush(); std::string path; + int64_t workerId; std::string tmpPath; FILE *file; std::vector buffer; @@ -168,6 +172,7 @@ class EdgeBucketWriter { EdgeBucketWriter &operator=(const EdgeBucketWriter &); void flush(unsigned int bucket); + std::string shardPath(unsigned int bucket) const; std::string dir; std::string shardId; @@ -175,6 +180,9 @@ class EdgeBucketWriter { size_t edgesPerBuffer; std::vector > buffers; std::vector files; + // Which buckets this writer has actually appended to, so flushAll() syncs only + // those rather than opening all bucketCount of them. + std::vector written; uint64_t edgeCount; bool closed; unsigned int currentPartition; diff --git a/src/linclust/KmerPartition.cpp b/src/linclust/KmerPartition.cpp index fa282a614..f95f1c1e3 100644 --- a/src/linclust/KmerPartition.cpp +++ b/src/linclust/KmerPartition.cpp @@ -290,10 +290,16 @@ uint64_t KmerBucketReader::countRecords(const std::string &dir, unsigned int par for (size_t i = 0; i < shards.size(); i++) { const size_t bytes = FileUtil::getFileSize(shards[i]); if (bytes % sizeof(KmerRecord) != 0) { - Debug(Debug::ERROR) << "Bucket " << shards[i] << " is " << bytes - << " bytes, not a whole number of k-mer records. " - << "It was probably written by an interrupted worker.\n"; - EXIT(EXIT_FAILURE); + // Counted down to the last whole record rather than refused. A torn + // tail is exactly what an interrupted worker leaves, and the map + // redoes that item into a *different* shard, so the records are not + // lost -- but the torn shard stays on disk, and making it fatal meant + // every later reduce of that partition died on it forever, with no + // recovery but deleting the file by hand. readPartitionAsPositions + // already stops at the last whole record for the same reason. + Debug(Debug::WARNING) << "Bucket " << shards[i] << " ends mid-record at " << bytes + << " bytes, as an interrupted worker leaves it; reading the " + << (bytes / sizeof(KmerRecord)) << " whole records it holds.\n"; } total += bytes / sizeof(KmerRecord); } diff --git a/src/linclust/alignparallel.cpp b/src/linclust/alignparallel.cpp index 0d0281d4e..8e52a799c 100644 --- a/src/linclust/alignparallel.cpp +++ b/src/linclust/alignparallel.cpp @@ -120,13 +120,6 @@ size_t mergePairCopies(std::vector &edges) { return out; } -// Which worker's edges count for each k-mer partition. -// -// The reduce's work queue is the authority: complete() keeps the first worker to -// record an item done, so exactly one is named per partition, and any block from -// another worker is a copy left by one that died before recording it. One queue -// per extraction wave, and wave w holds the partitions after all earlier waves', -// so appending them in wave order indexes by partition directly. // Which worker's edges count, per k-mer partition. // // A partition redone after a crash leaves the dead worker's edges on disk as well @@ -408,7 +401,7 @@ int alignparallel(int argc, const char **argv, const Command &command) { readBucket(edgeDir, static_cast(bucket), reduceAuthority, edges); const size_t raw = edges.size(); if (raw == 0) { - EdgeWriter empty(EdgeWriter::partitionPath(alnDir, static_cast(bucket))); + EdgeWriter empty(EdgeWriter::partitionPath(alnDir, static_cast(bucket)), workerId); empty.close(); return; } @@ -603,7 +596,7 @@ int alignparallel(int argc, const char **argv, const Command &command) { } } - EdgeWriter writer(EdgeWriter::partitionPath(alnDir, static_cast(bucket))); + EdgeWriter writer(EdgeWriter::partitionPath(alnDir, static_cast(bucket)), workerId); size_t kept = 0; for (size_t i = 0; i < edges.size(); i++) { if (survives[i]) { diff --git a/src/linclust/createrepdb.cpp b/src/linclust/createrepdb.cpp index 8edba4bc2..ad9050cb1 100644 --- a/src/linclust/createrepdb.cpp +++ b/src/linclust/createrepdb.cpp @@ -275,14 +275,6 @@ int createrepdb(int argc, const char **argv, const Command &command) { } - // Renamed only once complete: the pass-2 filter gate sizes itself from this - - // file's length, so a truncated one would silently be used as a short, - - // wrong sub-key to original-key map. - - FileUtil::move(keymapTmp.c_str(), mapFile.c_str()); - const int dbType = FileUtil::parseDbType(seqDb.c_str()); FileUtil::writeFile(repDb + ".dbtype", reinterpret_cast(&dbType), sizeof(int)); @@ -290,6 +282,18 @@ int createrepdb(int argc, const char **argv, const Command &command) { FileUtil::writeFile(repDb + "_h.dbtype", reinterpret_cast(&hdrType), sizeof(int)); + // The key map is renamed last, after the .dbtype files, because the workflow + // guards this whole stage on its existence. Publishing it first meant a death + // in the window before the .dbtype files were written left a marker saying the + // stage was finished over a database that could not be opened -- and every + // restart then skipped the stage and failed in the map, permanently. This is + // the same last-write-wins sentinel rule createdbparallel uses. + // + // The rename is also what makes the map itself safe to publish: the pass-2 + // filter gate sizes itself from this file's length, so a truncated one would + // silently be used as a short, wrong sub-key to original-key map. + FileUtil::move(keymapTmp.c_str(), mapFile.c_str()); + Debug(Debug::INFO) << "Wrote " << repDb << ": " << repCount << " sequences, " << seq.dataBytes << " data bytes, longest " << seq.maxLen << "\n"; return EXIT_SUCCESS; diff --git a/src/linclust/kmermatcherparallel.cpp b/src/linclust/kmermatcherparallel.cpp index 4885b5a9b..055523c6a 100644 --- a/src/linclust/kmermatcherparallel.cpp +++ b/src/linclust/kmermatcherparallel.cpp @@ -137,20 +137,6 @@ struct ShuffleManifest { return manifest; } - void requireMatches(const ShuffleManifest &other, const std::string &path) const { - if (entryCount != other.entryCount || partitionCount != other.partitionCount || - waveCount != other.waveCount || kmerSize != other.kmerSize) { - Debug(Debug::ERROR) - << "This worker derived a different k-mer shuffle than the one already in " - << "progress (" << path << "): " << partitionCount << " partitions, " - << waveCount << " waves, k=" << kmerSize << " over " << entryCount - << " sequences, against " << other.partitionCount << " partitions, " - << other.waveCount << " waves, k=" << other.kmerSize << " over " - << other.entryCount << " sequences.\n" - << "Every worker must run the same command line on the same database.\n"; - EXIT(EXIT_FAILURE); - } - } }; BaseMatrix *createSubstitutionMatrix(Parameters &par, int dbType) { @@ -294,9 +280,75 @@ int kmermatcherparallel(int argc, const char **argv, const Command &command) { << occupied << " B; reserving " << projectedEdgeBytes << " B for candidate edges\n"; } - const KmerShuffleSizing sizing = - deriveKmerShuffleSizing(info.entryCount, kmersPerSequence, par.scratchBudget, - persistentBytes, Util::computeMemory(par.splitMemoryLimit)); + if (FileUtil::directoryExists(kmerDir.c_str()) == false) { + FileUtil::makeDir(kmerDir.c_str()); + } + // Derived rather than passed, so every worker runs a byte-identical command line. + const std::string coordDir = kmerDir + "/coord"; + if (FileUtil::directoryExists(coordDir.c_str()) == false) { + FileUtil::makeDir(coordDir.c_str()); + } + const std::string manifestPath = coordDir + "/shuffle.info"; + + // The manifest is authoritative once it exists; the sizing is derived only to + // create it. + // + // The derivation reads how much scratch is already occupied, which by + // definition grows as the run proceeds -- a restart part-way through a waved + // pass sees a wave of k-mers and the edges written so far, derives a larger + // wave count from the smaller remaining budget, and every worker then fails + // requireMatches against the manifest wave 0 wrote. That made any run with a + // --scratch-budget unresumable, which is exactly the configuration waves exist + // for. Adopting the recorded sizing keeps the partitioning fixed for the life + // of the shuffle, which is the property that has to hold anyway: a partition + // is only self-contained if every wave agrees on P. + KmerShuffleSizing sizing; + { + FileLock manifestLock(coordDir + "/shuffle.lock"); + manifestLock.lock(); + if (FileUtil::fileExists(manifestPath.c_str()) == true) { + const ShuffleManifest existing = ShuffleManifest::read(manifestPath); + // Still checked, because these describe the *input* rather than the + // filesystem: a different database or k-mer length means the workers + // are not running the same job at all. + if (existing.entryCount != info.entryCount || + existing.kmerSize != static_cast(par.kmerSize)) { + Debug(Debug::ERROR) + << "The shuffle already in progress (" << manifestPath << ") covers " + << existing.entryCount << " sequences at k=" << existing.kmerSize + << ", but this worker was given " << info.entryCount << " sequences at k=" + << par.kmerSize << ".\n" + << "Every worker must run the same command line on the same database.\n"; + EXIT(EXIT_FAILURE); + } + // Assembled from the manifest rather than derived. deriveKmerShuffleSizing + // must not even be *called* here: by the time a run is resumed the + // occupied scratch can exceed the budget on its own -- the k-mers and + // edges already written are what fills it -- and the derivation treats + // that as fatal. The recorded layout is the answer, not something to + // check the derivation against. + sizing.totalKmerBytes = + info.entryCount * kmersPerSequence * sizeof(KmerRecord); + sizing.partitionCount = static_cast(existing.partitionCount); + sizing.waveCount = static_cast(existing.waveCount); + sizing.bytesPerWave = + (sizing.totalKmerBytes + sizing.waveCount - 1) / sizing.waveCount; + sizing.bytesPerPartition = + (sizing.bytesPerWave + sizing.partitionCount - 1) / sizing.partitionCount; + } else { + sizing = deriveKmerShuffleSizing(info.entryCount, kmersPerSequence, par.scratchBudget, + persistentBytes, + Util::computeMemory(par.splitMemoryLimit)); + ShuffleManifest manifest; + manifest.entryCount = info.entryCount; + manifest.partitionCount = sizing.partitionCount; + manifest.waveCount = sizing.waveCount; + manifest.kmerSize = static_cast(par.kmerSize); + KmerBucketWriter::createLayout(kmerDir, sizing.partitionCount); + manifest.write(manifestPath); + } + manifestLock.unlock(); + } Debug(Debug::INFO) << "K-mer shuffle: " << sizing.partitionCount << " partitions, " << sizing.waveCount << " wave(s), " << sizing.totalKmerBytes << " k-mer bytes, " << sizing.bytesPerPartition << " bytes per partition\n"; @@ -332,33 +384,6 @@ int kmermatcherparallel(int argc, const char **argv, const Command &command) { EXIT(EXIT_FAILURE); } - if (FileUtil::directoryExists(kmerDir.c_str()) == false) { - FileUtil::makeDir(kmerDir.c_str()); - } - // Derived rather than passed, so every worker runs a byte-identical command line. - const std::string coordDir = kmerDir + "/coord"; - if (FileUtil::directoryExists(coordDir.c_str()) == false) { - FileUtil::makeDir(coordDir.c_str()); - } - - ShuffleManifest manifest; - manifest.entryCount = info.entryCount; - manifest.partitionCount = sizing.partitionCount; - manifest.waveCount = sizing.waveCount; - manifest.kmerSize = static_cast(par.kmerSize); - const std::string manifestPath = coordDir + "/shuffle.info"; - { - FileLock manifestLock(coordDir + "/shuffle.lock"); - manifestLock.lock(); - if (FileUtil::fileExists(manifestPath.c_str()) == true) { - manifest.requireMatches(ShuffleManifest::read(manifestPath), manifestPath); - } else { - KmerBucketWriter::createLayout(kmerDir, sizing.partitionCount); - manifest.write(manifestPath); - } - manifestLock.unlock(); - } - SharedCounter workerCounter(coordDir + "/worker.counter"); const int64_t workerId = workerCounter.fetchAdd(); const uint64_t sequencesPerItem = deriveSequencesPerItem(info.entryCount); diff --git a/src/linclust/kmerreduceparallel.cpp b/src/linclust/kmerreduceparallel.cpp index 0c29c944f..daba301b0 100644 --- a/src/linclust/kmerreduceparallel.cpp +++ b/src/linclust/kmerreduceparallel.cpp @@ -78,7 +78,19 @@ size_t readPartitionAsPositions(const std::string &kmerDir, unsigned int partiti std::vector block(blockRecords); size_t filled = 0; for (size_t i = 0; i < shards.size(); i++) { - FILE *file = FileUtil::openFileOrDie(shards[i].c_str(), "rb", true); + // Not openFileOrDie: a worker whose lease lapsed can still be here while + // the workers that finished the wave unlink its shards. Its results are + // discarded by block header regardless, so the shard vanishing under it is + // a race it should survive rather than a reason to fail the whole stage. + FILE *file = fopen(shards[i].c_str(), "rb"); + if (file == NULL) { + if (errno == ENOENT) { + continue; + } + Debug(Debug::ERROR) << "Cannot open k-mer bucket " << shards[i] << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } while (true) { const size_t got = fread(block.data(), sizeof(KmerRecord), blockRecords, file); if (got == 0) { @@ -553,12 +565,12 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { // one. Every worker that observes the queue finished reaches this and // unlinks, hence the ENOENT tolerance below. // - // The one reader this does not account for is a worker whose lease lapsed - // while it was still inside reducePartition: its item was redone and - // recorded by someone else, so the queue reads finished while it is still - // opening shards. Its own edges are already discarded by block header, so - // the outcome is a spurious open failure rather than a wrong answer, and - // the heartbeat makes a lapse unlikely to begin with. + // A worker whose lease lapsed while still inside reducePartition can be + // reading a shard as another unlinks it. That costs nothing: its item was + // redone and recorded by someone else, so its edges are discarded by block + // header anyway. What it must not do is turn into a stage failure, so + // readPartitionAsPositions treats a shard that disappears under it as + // empty rather than as an error. for (unsigned int p = waveFrom; p < waveTo; p++) { const std::vector shards = KmerBucketReader::shardFiles(kmerDir, p); for (size_t i = 0; i < shards.size(); i++) { diff --git a/src/util/createdbparallel.cpp b/src/util/createdbparallel.cpp index 978733228..62e030a5e 100644 --- a/src/util/createdbparallel.cpp +++ b/src/util/createdbparallel.cpp @@ -31,6 +31,7 @@ #include "FastSort.h" #include "FileUtil.h" #include "KSeqWrapper.h" +#include "KmerPartition.h" #include "LengthRankedPlan.h" #include "ParallelCoordination.h" #include "Parameters.h" @@ -421,20 +422,33 @@ void allocateFile(const std::string &path, size_t size) { // Runs a queue to completion with every thread of this worker claiming its own // items. Leases mean a worker that dies mid-item only delays that item; another // worker picks it up once the lease expires. -// The claim is recorded against the *process*, not the thread, because that is -// the unit that dies: if this worker is killed, every item its threads held must -// become re-claimable together once the lease expires. Passing a thread index -// here instead would also make identities collide across worker processes, since -// every process numbers its threads from zero. +// +// Every thread drains the queue, so each needs an id of its own. +// +// claim()'s lease-expiry test and renew()'s ownership test are both keyed on +// (item, worker). With one id shared by all threads of a process, a thread whose +// lease lapsed could have its item re-claimed by a *sibling* thread, and the first +// thread's heartbeat would then keep renewing a record the sibling now owns -- +// both running the same item with the queue providing no exclusion at all. Only +// the idempotence of the bodies here made that harmless, which is not a property +// to rely on silently. +// +// workerId * threads + threadNum is unique across the run because worker ids come +// from a shared counter and every process uses the same thread count. template void runQueue(WorkQueue &queue, int threads, int64_t workerId, Body body) { bool stalled = false; #pragma omp parallel num_threads(threads) { + int threadNum = 0; +#ifdef OPENMP + threadNum = omp_get_thread_num(); +#endif + const int64_t threadWorkerId = workerId * static_cast(threads) + threadNum; // drain() rather than a plain claim loop: it goes back to claiming after // waiting, so items abandoned by a crashed worker are picked up once their // leases expire instead of being waited on by everyone forever. - if (queue.drain(workerId, body) == false) { + if (queue.drain(threadWorkerId, body) == false) { #pragma omp critical stalled = true; } @@ -465,6 +479,27 @@ int createdbparallel(int argc, const char **argv, const Command &command) { Debug(Debug::ERROR) << "File " << filenames[i] << " is a directory\n"; EXIT(EXIT_FAILURE); } + // Named here rather than left to the planner. Compressed input is read as + // plain text, finds no '>' and plans zero sequences, so the run used to die + // several stages later with "The input files have no entry" -- true, and + // useless. createdb decompresses; this cannot, because both passes address + // the file by byte offset and a gzip stream has no seekable record + // boundaries. + unsigned char magic[2] = {0, 0}; + FILE *probe = fopen(filenames[i].c_str(), "rb"); + if (probe != NULL) { + const size_t got = fread(magic, 1, sizeof(magic), probe); + fclose(probe); + if (got == sizeof(magic) + && ((magic[0] == 0x1f && magic[1] == 0x8b) + || (magic[0] == 'B' && magic[1] == 'Z'))) { + Debug(Debug::ERROR) << filenames[i] << " is compressed, which this command cannot " + << "read: both passes address the input by byte offset, and a " + << "compressed stream has no seekable record boundaries. " + << "Decompress it first.\n"; + EXIT(EXIT_FAILURE); + } + } } const std::string hdrDataFile = dataFile + "_h"; @@ -520,6 +555,26 @@ int createdbparallel(int argc, const char **argv, const Command &command) { Debug(Debug::ERROR) << "The input files have no entry\n"; EXIT(EXIT_FAILURE); } + // Refused rather than wrapped. Keys are dense, so the last one is + // seqCount - 1, and two ceilings apply: DBKeyType, which is a uint32_t + // unless the build sets MMSEQS_INT64_IDS -- the CMake default is 0 -- + // and the 48-bit id field of KmerRecord. Past either, distinct + // sequences alias to one key and the reduce groups unrelated sequences + // together, silently. The 32-bit ceiling bites three orders of + // magnitude below the scale this command exists for, so it is worth + // saying which limit was hit. + const uint64_t keyCeiling = + std::min(KmerRecord::MAX_ID, static_cast(DB_KEY_INVALID) - 1); + if (totals.seqCount - 1 > keyCeiling) { + Debug(Debug::ERROR) + << "The input holds " << totals.seqCount << " sequences, past the " + << (keyCeiling + 1) << " this build can key.\n" + << (static_cast(DB_KEY_INVALID) - 1 < KmerRecord::MAX_ID + ? "Rebuild with -DMMSEQS_INT64_IDS=1; the default build uses 32-bit " + "database keys.\n" + : "The k-mer record carries a 48-bit id, which is the hard limit.\n"); + EXIT(EXIT_FAILURE); + } for (size_t i = 0; i < plans.size(); i++) { plans[i].write(chunkPlanPath(coordDir, plans[i].chunkIdx)); } diff --git a/src/workflow/LinclustParallel.cpp b/src/workflow/LinclustParallel.cpp index 4ceca5682..1d83dc0e6 100644 --- a/src/workflow/LinclustParallel.cpp +++ b/src/workflow/LinclustParallel.cpp @@ -34,6 +34,9 @@ void setLinclustParallelWorkflowDefaults(Parameters *p) { p->covMode = Parameters::COV_MODE_TARGET; p->evalThr = 0.001; p->seqIdThr = 0.9; + // As linclust does (Linclust.cpp:18). Without it the alignment downgrades to + // SCORE_COV at --min-seq-id 0, where stock stays SCORE_COV_SEQID. + p->alignmentMode = Parameters::ALIGNMENT_MODE_SCORE_COV_SEQID; } int linclustparallel(int argc, const char **argv, const Command &command) { @@ -41,6 +44,23 @@ int linclustparallel(int argc, const char **argv, const Command &command) { setLinclustParallelWorkflowDefaults(&par); par.parseParameters(argc, argv, command, true, 0, 0); + // Only the coverage modes whose clustering this pipeline actually implements. + // + // For a symmetric --cov-mode, linclust switches to SET_COVER *and* enables the + // count table's extra grouping rounds (Linclust.cpp:91-109). greedycluster + // implements only the length-ordered greedy and the reduce runs no count-table + // rounds, so accepting mode 0 would apply this pipeline's thresholds to a + // different algorithm and return a silently different clustering -- measured + // at 101 clusters against stock's 117 on a 389-sequence input, exit 0. + if (par.covMode != Parameters::COV_MODE_TARGET && par.covMode != Parameters::COV_MODE_QUERY) { + Debug(Debug::ERROR) << "--cov-mode " << par.covMode << " is not supported.\n" + << "linclust selects SET_COVER clustering and the count-table grouping " + << "rounds for symmetric coverage modes; this pipeline implements " + << "neither, so the result would differ from linclust without saying " + << "so. Use --cov-mode 1 (target) or 2 (query).\n"; + EXIT(EXIT_FAILURE); + } + std::string tmpDir = par.db3; std::string hash = SSTR(par.hashParameter(command.databases, par.filenames, par.linclustparallelworkflow)); @@ -52,9 +72,13 @@ int linclustparallel(int argc, const char **argv, const Command &command) { par.filenames.push_back(tmpDir); CommandCaller cmd; - // --runner, not --mpi-runner: this only has to start one worker per node, and - // the workers coordinate through files rather than communicating. - cmd.addVariable("RUNNER", par.workerRunner.c_str()); + // WORKER_RUNNER, not RUNNER: ten existing workflows set RUNNER from the *MPI* + // --mpi-runner, and Parameters.cpp seeds par.runner from getenv("RUNNER"), so + // reusing the name would hand every stage sub-process an MPI runner it never + // asked for. --runner itself is separate from --mpi-runner because it only has + // to start one worker per node; the workers coordinate through files. + cmd.addVariable("WORKER_RUNNER", par.workerRunner.c_str()); + cmd.addVariable("VERBOSITY", par.createParameterString(par.onlyverbosity).c_str()); cmd.addVariable("THREADS", SSTR(par.threads).c_str()); cmd.addVariable("MIN_SEQ_ID", SSTR(par.seqIdThr).c_str()); cmd.addVariable("COV", SSTR(par.covThr).c_str()); From e957f2dc8638515fa07ca472816b283fed57598d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Mon, 3 Aug 2026 12:38:33 +0000 Subject: [PATCH 18/27] Revert unnecessary fsync fix (out of scope, requires too much resources). MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Björn Buschkämper --- src/linclust/CandidateEdge.cpp | 77 +++++----------------------------- src/linclust/CandidateEdge.h | 3 -- 2 files changed, 10 insertions(+), 70 deletions(-) diff --git a/src/linclust/CandidateEdge.cpp b/src/linclust/CandidateEdge.cpp index fba2f8700..8777c141c 100644 --- a/src/linclust/CandidateEdge.cpp +++ b/src/linclust/CandidateEdge.cpp @@ -11,33 +11,8 @@ #include #include -#include #include -namespace { - -// Makes a rename durable. Renaming into place is atomic against other readers but -// not against losing the node: the directory entry lives in the parent, so without -// this the file can come back missing or empty after a crash the work queue has -// already recorded as a completed item. -void syncParentDirectory(const std::string &path) { - const size_t slash = path.find_last_of('/'); - const std::string dir = slash == std::string::npos ? std::string(".") : path.substr(0, slash); - const int fd = open(dir.c_str(), O_RDONLY); - if (fd < 0) { - Debug(Debug::ERROR) << "Cannot open " << dir << " to flush it: " << strerror(errno) << "\n"; - EXIT(EXIT_FAILURE); - } - if (fsync(fd) != 0) { - Debug(Debug::ERROR) << "Cannot flush " << dir << ": " << strerror(errno) << "\n"; - close(fd); - EXIT(EXIT_FAILURE); - } - close(fd); -} - -} // namespace - std::string EdgeWriter::partitionPath(const std::string &dir, unsigned int partition) { return dir + "/p" + SSTR(partition) + ".edges"; } @@ -108,17 +83,9 @@ void EdgeWriter::close() { // can tell "this partition was reduced and had nothing" from "this // partition was never reduced". // - // fsync before the rename, and the directory after it, because the work - // queue that vouches for this file *is* fsynced when the item is completed. - // Without this the durability runs the wrong way round: a node lost after - // the rename can replay the metadata without the data extents, leaving a - // zero-filled bucket that the queue records as DONE, so nobody redoes it - // and greedycluster reads it as "no edges" in silence. - if (fflush(file) != 0 || fsync(fileno(file)) != 0) { - Debug(Debug::ERROR) << "Cannot flush edge file " << tmpPath << ": " << strerror(errno) - << "\n"; - EXIT(EXIT_FAILURE); - } + // Not fsynced. Losing the node mid-stage is out of scope: the pipeline + // resumes between stages, not within one, so a stage interrupted that way + // is re-run from its start rather than trusted from its work queue. if (fclose(file) != 0) { Debug(Debug::ERROR) << "Cannot close edge file " << tmpPath << ": " << strerror(errno) << "\n"; @@ -130,7 +97,6 @@ void EdgeWriter::close() { << strerror(errno) << "\n"; EXIT(EXIT_FAILURE); } - syncParentDirectory(path); } else { const std::string tmp = path + ".w" + SSTR(workerId); FILE *empty = fopen(tmp.c_str(), "wb"); @@ -144,7 +110,6 @@ void EdgeWriter::close() { << strerror(errno) << "\n"; EXIT(EXIT_FAILURE); } - syncParentDirectory(path); } } @@ -178,7 +143,6 @@ EdgeBucketWriter::EdgeBucketWriter(const std::string &dir, unsigned int bucketCo edgesPerBuffer = std::max(perBucket, 64); buffers.resize(bucketCount); files.assign(bucketCount, NULL); - written.assign(bucketCount, false); } EdgeBucketWriter::~EdgeBucketWriter() { @@ -242,7 +206,6 @@ void EdgeBucketWriter::flush(unsigned int bucket) { EXIT(EXIT_FAILURE); } files[bucket] = NULL; - written[bucket] = true; buffer.clear(); } @@ -266,37 +229,17 @@ void EdgeBucketWriter::append(unsigned int bucket, const CandidateEdge &edge) { edgeCount++; } +// Pushes every buffered edge to the OS before the caller marks the work item done. +// +// Deliberately not fsynced. That would make the data durable against losing the +// node, which the queue's own completion record already is -- but resume here is +// between stages, not within one, so an interrupted stage is re-run from its start +// rather than trusted item by item. Syncing each touched shard once per item cost +// real traffic on a parallel filesystem for a guarantee nothing consumes. void EdgeBucketWriter::flushAll() { for (unsigned int b = 0; b < bucketCount; b++) { flush(b); } - // Then make what was written durable, before the caller marks the work item - // done. The queue fsyncs its own completion record, so without this the record - // that an item finished outlives the data it vouches for: a node lost here - // brings the shard back short or zero-filled, and because the item reads DONE - // nobody redoes it. - // - // Once per item rather than once per flush -- a flush is a few hundred KB at - // large bucket counts, and syncing each would dominate the stage. - for (unsigned int b = 0; b < bucketCount; b++) { - if (written[b] == false) { - continue; - } - const std::string path = shardPath(b); - const int fd = ::open(path.c_str(), O_WRONLY | O_APPEND); - if (fd < 0 || fsync(fd) != 0) { - Debug(Debug::ERROR) << "Cannot flush edge bucket " << path << " to disk: " - << strerror(errno) << "\n"; - if (fd >= 0) { - ::close(fd); - } - EXIT(EXIT_FAILURE); - } - ::close(fd); - // The shard's own directory, because a shard created during this item is - // only reachable once its directory entry is durable too. - syncParentDirectory(path); - } } void EdgeBucketWriter::close() { diff --git a/src/linclust/CandidateEdge.h b/src/linclust/CandidateEdge.h index 006643851..ba63a076e 100644 --- a/src/linclust/CandidateEdge.h +++ b/src/linclust/CandidateEdge.h @@ -180,9 +180,6 @@ class EdgeBucketWriter { size_t edgesPerBuffer; std::vector > buffers; std::vector files; - // Which buckets this writer has actually appended to, so flushAll() syncs only - // those rather than opening all bucketCount of them. - std::vector written; uint64_t edgeCount; bool closed; unsigned int currentPartition; From 0b78481e96ccfca1605f1d4f901a05193806c8be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Mon, 3 Aug 2026 13:09:08 +0000 Subject: [PATCH 19/27] Fix silent error in merge cluster parallel. Fix memory budget over-allocation during translate keys. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Björn Buschkämper --- src/linclust/mergeclusterparallel.cpp | 13 ++++++++++++- src/linclust/translatekeys.cpp | 20 ++++++++++++++++---- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/linclust/mergeclusterparallel.cpp b/src/linclust/mergeclusterparallel.cpp index bb53e42cf..a4588cb6c 100644 --- a/src/linclust/mergeclusterparallel.cpp +++ b/src/linclust/mergeclusterparallel.cpp @@ -154,9 +154,20 @@ std::vector readBucket(const std::string &prefix, unsigned int bucket) { return out; } const size_t bytes = FileUtil::getFileSize(p); - if (bytes == 0 || bytes % sizeof(Pair) != 0) { + if (bytes == 0) { return out; } + if (bytes % sizeof(Pair) != 0) { + // Not silently skipped. These buckets are written by this same command a + // moment earlier, in whole Pair units, so a partial one means a failed + // write or a truncated filesystem -- not something the input can cause. + // Returning empty dropped every cluster in the bucket's key range from the + // merged output, which is a smaller clustering that still looks valid. + Debug(Debug::ERROR) << "Bucket " << p << " is " << bytes << " bytes, not a whole number of " + << sizeof(Pair) << "-byte pairs. It was truncated after this stage " + << "wrote it; remove the working directory and re-run the merge.\n"; + EXIT(EXIT_FAILURE); + } out.resize(bytes / sizeof(Pair)); FILE *f = FileUtil::openFileOrDie(p.c_str(), "rb", true); if (fread(out.data(), sizeof(Pair), out.size(), f) != out.size()) { diff --git a/src/linclust/translatekeys.cpp b/src/linclust/translatekeys.cpp index f20b28aea..efad4b630 100644 --- a/src/linclust/translatekeys.cpp +++ b/src/linclust/translatekeys.cpp @@ -353,9 +353,18 @@ int translatekeys(int argc, const char **argv, const Command &command) { const std::string outTsv = par.db3; const int threads = std::max(1, par.threads); - // Sized so one slice of accessions fits the memory budget. An accession is - // ~30 bytes plus the std::string that holds it, so 64 bytes per key is a safe - // working figure; being wrong only changes how many buckets are used. + // Sized so all threads' resident state together fits the memory budget. + // + // Every thread works a bucket of its own and holds that bucket's accession + // slice, so the budget has to be divided by the thread count -- sizing one + // slice against the whole budget, as this used to, overran it by exactly the + // thread count and turned a 64-thread run into ~18x its --split-memory-limit. + // + // 192 bytes per key, not 64: an accession is ~30 bytes plus the std::string + // that holds it, and pass 3 keeps three of these alive at once -- the slice, + // the sorted TranslatedRow vector (which carries the member accession), and + // the output buffer being assembled. Being wrong only changes how many buckets + // are used. uint64_t keyCount = 0; { LookupCursor probe(lookupFile); @@ -375,8 +384,11 @@ int translatekeys(int argc, const char **argv, const Command &command) { } const uint64_t targetBytes = std::max(Util::computeMemory(par.splitMemoryLimit) / 8, 1ULL * 1024 * 1024); + const uint64_t bytesPerKey = 192; + const uint64_t perThreadBudget = + std::max(targetBytes / static_cast(threads), 1ULL * 1024 * 1024); unsigned int buckets = 1; - while (buckets < 65536 && (keyCount / buckets) * 64 > targetBytes) { + while (buckets < 65536 && (keyCount / buckets) * bytesPerKey > perThreadBudget) { buckets *= 2; } const uint64_t span = (keyCount + buckets - 1) / buckets; From f5b53deae86853ee386484f139ad58108999e72b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Tue, 4 Aug 2026 09:08:23 +0000 Subject: [PATCH 20/27] More fixes. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Björn Buschkämper --- src/commons/LengthRankedPlan.cpp | 35 ++++++++-- src/linclust/CandidateEdge.cpp | 23 ++++++- src/linclust/KmerPartition.cpp | 44 +++++++++++-- src/linclust/greedycluster.cpp | 2 +- src/linclust/mergeclusterparallel.cpp | 32 +++++---- src/linclust/translatecluster.cpp | 29 +++++---- src/linclust/translatekeys.cpp | 94 ++++++++++++++++++++------- src/test/TestKmerPartition.cpp | 11 +++- src/test/TestParallelCoordination.cpp | 5 +- 9 files changed, 208 insertions(+), 67 deletions(-) diff --git a/src/commons/LengthRankedPlan.cpp b/src/commons/LengthRankedPlan.cpp index ecaa99a17..65f5b692e 100644 --- a/src/commons/LengthRankedPlan.cpp +++ b/src/commons/LengthRankedPlan.cpp @@ -6,7 +6,11 @@ #include #include +#include #include +#include +#include +#include namespace { @@ -29,13 +33,30 @@ struct FileHeader { void writeBlock(const std::string &path, const FileHeader &header, const void *entries, size_t entryBytes) { // Write to a temporary and rename, so a worker that dies mid-write leaves no - // truncated file that a later reader would mistake for a complete one. - // Tagged with the pid: two workers redoing the same chunk would otherwise - // both open the same .tmp, and one could truncate the other's in-flight write - // before renaming the hole-ridden result into place. The rename itself is - // atomic, so a private temp makes the whole publish atomic. - std::string tmp = path + ".tmp." + SSTR(getpid()); - FILE *file = FileUtil::openAndDelete(tmp.c_str(), "wb"); + // truncated file that a later reader would mistake for a complete one. Two + // workers redoing the same chunk must not share the temporary path, or one + // can truncate the other's in-flight write and then rename the hole-ridden + // result into place. The rename itself is atomic, so a private temp makes the + // whole publish atomic. + // + // mkstemp, not getpid(): this path is on the shared filesystem and pids are + // node-local, so two workers on different nodes drawing the same pid -- which + // a homogeneous allocation makes routine -- would collide on the same name. + std::string tmp = path + ".tmp.XXXXXX"; + std::vector tmpName(tmp.begin(), tmp.end()); + tmpName.push_back('\0'); + const int tmpFd = mkstemp(&tmpName[0]); + if (tmpFd < 0) { + Debug(Debug::ERROR) << "Cannot create a temporary next to " << path << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + tmp.assign(&tmpName[0]); + FILE *file = fdopen(tmpFd, "wb"); + if (file == NULL) { + Debug(Debug::ERROR) << "Cannot open " << tmp << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } if (fwrite(&header, sizeof(FileHeader), 1, file) != 1) { Debug(Debug::ERROR) << "Cannot write header to " << tmp << "\n"; EXIT(EXIT_FAILURE); diff --git a/src/linclust/CandidateEdge.cpp b/src/linclust/CandidateEdge.cpp index 8777c141c..a7c4ff27e 100644 --- a/src/linclust/CandidateEdge.cpp +++ b/src/linclust/CandidateEdge.cpp @@ -255,18 +255,37 @@ void EdgeBucketWriter::close() { std::vector EdgeBucketWriter::shardFiles(const std::string &dir, unsigned int bucket) { const std::string path = bucketDir(dir, bucket); std::vector shards; + // createLayout pre-creates every partition/bucket directory, so a directory + // that cannot be opened is a real failure, not a legitimately empty one -- + // treating it as empty let a worker record the item done having read nothing, + // silently dropping every record in it. An empty partition is an *existing*, + // readable directory holding no shards. DIR *handle = opendir(path.c_str()); if (handle == NULL) { - return shards; // a bucket nothing was written to is empty, not an error + // errno captured before the Debug chain, which does its own I/O and would + // otherwise overwrite it. + const int err = errno; + Debug(Debug::ERROR) << "Cannot open edge bucket directory " << path << ": " << strerror(err) << "\n"; + EXIT(EXIT_FAILURE); } struct dirent *entry; + errno = 0; while ((entry = readdir(handle)) != NULL) { const std::string name = entry->d_name; if (name.size() > 6 && name.compare(name.size() - 6, 6, ".edges") == 0) { shards.push_back(path + "/" + name); } } - closedir(handle); + const int readErr = errno; + if (readErr != 0) { + Debug(Debug::ERROR) << "Cannot read " << path << ": " << strerror(readErr) << "\n"; + EXIT(EXIT_FAILURE); + } + if (closedir(handle) != 0) { + const int closeErr = errno; + Debug(Debug::ERROR) << "Cannot close " << path << ": " << strerror(closeErr) << "\n"; + EXIT(EXIT_FAILURE); + } std::sort(shards.begin(), shards.end()); return shards; } diff --git a/src/linclust/KmerPartition.cpp b/src/linclust/KmerPartition.cpp index f95f1c1e3..af05091f3 100644 --- a/src/linclust/KmerPartition.cpp +++ b/src/linclust/KmerPartition.cpp @@ -105,6 +105,19 @@ KmerShuffleSizing deriveKmerShuffleSizing(uint64_t sequenceCount, unsigned int k } sizing.bytesPerWave = divideRoundingUp(sizing.totalKmerBytes, sizing.waveCount); + // Sized against totalKmerBytes, NOT bytesPerWave. + // + // A wave re-extracts every k-mer and keeps only its own contiguous slice of + // partition space, so a partition that a wave owns receives *all* of its + // k-mers -- there is no such thing as a partial partition. A partition + // therefore holds totalKmerBytes / P regardless of the wave count; what waves + // divide is how many partitions are on disk at once, which is why + // bytesPerWave is still the right figure for the scratch budget. + // + // Deriving P from bytesPerWave made P a factor of waveCount too small and + // reported a per-partition size the same factor too low, so a reduce worker + // held waveCount times the memory the limit allowed. Measured at 100M with + // two waves: reported 1.575 GB per partition against an actual 3.15 GB. if (workerMemoryBytes == 0) { sizing.partitionCount = 1; } else { @@ -117,7 +130,7 @@ KmerShuffleSizing deriveKmerShuffleSizing(uint64_t sequenceCount, unsigned int k // --split-memory-limit 0 on a 2 TB node at 1e11 that derived P = 32, a // 1.58 TB partition and ~3 TB of arrays. sizing.partitionCount = roundUpToPowerOfTwo( - divideRoundingUp(sizing.bytesPerWave * REDUCE_MEMORY_FACTOR, workerMemoryBytes)); + divideRoundingUp(sizing.totalKmerBytes * REDUCE_MEMORY_FACTOR, workerMemoryBytes)); } // Both are powers of two, so this makes the wave count a divisor of P. sizing.partitionCount = std::max(sizing.partitionCount, sizing.waveCount); @@ -132,7 +145,7 @@ KmerShuffleSizing deriveKmerShuffleSizing(uint64_t sequenceCount, unsigned int k << "waves are used.\n"; EXIT(EXIT_FAILURE); } - sizing.bytesPerPartition = divideRoundingUp(sizing.bytesPerWave, sizing.partitionCount); + sizing.bytesPerPartition = divideRoundingUp(sizing.totalKmerBytes, sizing.partitionCount); return sizing; } @@ -206,10 +219,18 @@ void KmerBucketWriter::flush(unsigned int partition) { << partition << " of " << dir << ": " << strerror(errno) << "\n"; EXIT(EXIT_FAILURE); } - buffer.clear(); - // Released immediately: see the note in flush() above. - fclose(files[partition]); + // Checked, and before the buffer is dropped: buffered I/O can report ENOSPC, + // a quota, or a remote filesystem error only at close, and the queue marks the + // item done straight after flushAll(). An unchecked close here let an item be + // recorded complete with k-mers missing, which the reduce then reads as "this + // k-mer had no partner" -- a wrong clustering with no diagnostic. + if (fclose(files[partition]) != 0) { + Debug(Debug::ERROR) << "Cannot close k-mer bucket " << partition << " of " << dir << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } files[partition] = NULL; + buffer.clear(); } void KmerBucketWriter::append(unsigned int partition, const KmerRecord &record) { @@ -265,12 +286,21 @@ void KmerBucketWriter::close() { std::vector KmerBucketReader::shardFiles(const std::string &dir, unsigned int partition) { const std::string path = KmerBucketWriter::partitionDir(dir, partition); std::vector shards; + // createLayout pre-creates every partition/bucket directory, so a directory + // that cannot be opened is a real failure, not a legitimately empty one -- + // treating it as empty let a worker record the item done having read nothing, + // silently dropping every record in it. An empty partition is an *existing*, + // readable directory holding no shards. DIR *handle = opendir(path.c_str()); if (handle == NULL) { - // A partition no worker wrote to is empty, not an error. - return shards; + // errno captured before the Debug chain, which does its own I/O and would + // otherwise overwrite it. + const int err = errno; + Debug(Debug::ERROR) << "Cannot open k-mer partition directory " << path << ": " << strerror(err) << "\n"; + EXIT(EXIT_FAILURE); } struct dirent *entry; + errno = 0; while ((entry = readdir(handle)) != NULL) { const std::string name = entry->d_name; if (name.size() > 6 && name.compare(name.size() - 6, 6, ".kmers") == 0) { diff --git a/src/linclust/greedycluster.cpp b/src/linclust/greedycluster.cpp index d6744e13b..58c1c6114 100644 --- a/src/linclust/greedycluster.cpp +++ b/src/linclust/greedycluster.cpp @@ -308,7 +308,7 @@ int greedycluster(int argc, const char **argv, const Command &command) { } for (int t = 0; t < par.threads; t++) { if (threadBuffers[t].empty() == false) { - fwrite(threadBuffers[t].data(), 1, threadBuffers[t].size(), out); + writeAllOrDie(threadBuffers[t].data(), threadBuffers[t].size(), out, outFile); } } singletonCount += blockSingletons; diff --git a/src/linclust/mergeclusterparallel.cpp b/src/linclust/mergeclusterparallel.cpp index a4588cb6c..6d9802579 100644 --- a/src/linclust/mergeclusterparallel.cpp +++ b/src/linclust/mergeclusterparallel.cpp @@ -64,7 +64,7 @@ class BucketWriter { public: BucketWriter(const std::string &prefix, unsigned int buckets, size_t bufferPairs = 1 << 16) : prefix(prefix), buckets(buckets), bufferPairs(bufferPairs), closed(false) { - files.assign(buckets, NULL); + opened.assign(buckets, false); this->buffers.resize(buckets); } ~BucketWriter() { close(); } @@ -86,10 +86,6 @@ class BucketWriter { closed = true; for (unsigned int b = 0; b < buckets; b++) { flush(b); - if (files[b] != NULL) { - fclose(files[b]); - files[b] = NULL; - } } } @@ -102,19 +98,27 @@ class BucketWriter { if (buffers[bucket].empty()) { return; } - if (files[bucket] == NULL) { - files[bucket] = fopen(path(prefix, bucket).c_str(), "wb"); - if (files[bucket] == NULL) { - Debug(Debug::ERROR) << "Cannot open " << path(prefix, bucket) << ": " - << strerror(errno) << "\n"; - EXIT(EXIT_FAILURE); - } + // Append-and-close per flush: one descriptor per bucket can reach the + // derived bucket count, and the close is where a buffered or remote + // filesystem first reports ENOSPC or a quota. Losing whole records keeps + // the file record-aligned, so downstream never notices the truncation. + FILE *file = fopen(path(prefix, bucket).c_str(), opened[bucket] ? "ab" : "wb"); + if (file == NULL) { + Debug(Debug::ERROR) << "Cannot open " << path(prefix, bucket) << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); } - if (fwrite(buffers[bucket].data(), sizeof(Pair), buffers[bucket].size(), files[bucket]) != + opened[bucket] = true; + if (fwrite(buffers[bucket].data(), sizeof(Pair), buffers[bucket].size(), file) != buffers[bucket].size()) { Debug(Debug::ERROR) << "Cannot write bucket " << bucket << ": " << strerror(errno) << "\n"; EXIT(EXIT_FAILURE); } + if (fclose(file) != 0) { + Debug(Debug::ERROR) << "Cannot close " << path(prefix, bucket) << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } buffers[bucket].clear(); } @@ -122,7 +126,7 @@ class BucketWriter { unsigned int buckets; size_t bufferPairs; std::vector > buffers; - std::vector files; + std::vector opened; bool closed; }; diff --git a/src/linclust/translatecluster.cpp b/src/linclust/translatecluster.cpp index eca1c9636..6b81e987d 100644 --- a/src/linclust/translatecluster.cpp +++ b/src/linclust/translatecluster.cpp @@ -55,7 +55,7 @@ class Buckets { public: Buckets(const std::string &prefix, unsigned int count, size_t bufferPairs = 1 << 16) : prefix(prefix), count(count), bufferPairs(bufferPairs), closed(false) { - files.assign(count, NULL); + opened.assign(count, false); buffers.resize(count); } ~Buckets() { close(); } @@ -73,7 +73,6 @@ class Buckets { closed = true; for (unsigned int b = 0; b < count; b++) { flush(b); - if (files[b] != NULL) { fclose(files[b]); files[b] = NULL; } } } @@ -100,19 +99,27 @@ class Buckets { private: void flush(unsigned int b) { if (buffers[b].empty()) return; - if (files[b] == NULL) { - files[b] = fopen(path(prefix, b).c_str(), "wb"); - if (files[b] == NULL) { - Debug(Debug::ERROR) << "Cannot open " << path(prefix, b) << ": " - << strerror(errno) << "\n"; - EXIT(EXIT_FAILURE); - } + // Append-and-close per flush: one descriptor per bucket can reach the + // derived bucket count, and the close is where a buffered or remote + // filesystem first reports ENOSPC or a quota. Losing whole records keeps + // the file record-aligned, so downstream never notices the truncation. + FILE *file = fopen(path(prefix, b).c_str(), opened[b] ? "ab" : "wb"); + if (file == NULL) { + Debug(Debug::ERROR) << "Cannot open " << path(prefix, b) << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); } - if (fwrite(buffers[b].data(), sizeof(Pair), buffers[b].size(), files[b]) != + opened[b] = true; + if (fwrite(buffers[b].data(), sizeof(Pair), buffers[b].size(), file) != buffers[b].size()) { Debug(Debug::ERROR) << "Cannot write bucket " << b << "\n"; EXIT(EXIT_FAILURE); } + if (fclose(file) != 0) { + Debug(Debug::ERROR) << "Cannot close " << path(prefix, b) << ": " << strerror(errno) + << "\n"; + EXIT(EXIT_FAILURE); + } buffers[b].clear(); } @@ -120,7 +127,7 @@ class Buckets { unsigned int count; size_t bufferPairs; std::vector > buffers; - std::vector files; + std::vector opened; bool closed; }; diff --git a/src/linclust/translatekeys.cpp b/src/linclust/translatekeys.cpp index efad4b630..b0342ea81 100644 --- a/src/linclust/translatekeys.cpp +++ b/src/linclust/translatekeys.cpp @@ -36,6 +36,8 @@ #include #include +#include + #ifdef OPENMP #include #endif @@ -71,6 +73,33 @@ struct TranslatedRow { std::string memberAccession; }; +// Removes every spill file belonging to a prefix, regardless of the thread and +// bucket counts the previous attempt used. +// +// Needed because a rerun only truncates the shards it happens to touch: pass 2 +// spills under .t. and assigns source buckets with +// schedule(dynamic), so a different schedule -- or a different --threads -- leaves +// shards from the earlier attempt on disk, and pass 3 reads every shard it finds. +// Both attempts' rows would then be emitted. +void removeSpillFiles(const std::string &prefix) { + const size_t slash = prefix.find_last_of('/'); + const std::string dir = slash == std::string::npos ? std::string(".") : prefix.substr(0, slash); + const std::string base = slash == std::string::npos ? prefix : prefix.substr(slash + 1); + DIR *handle = opendir(dir.c_str()); + if (handle == NULL) { + return; // nothing written yet + } + struct dirent *entry; + while ((entry = readdir(handle)) != NULL) { + const std::string name = entry->d_name; + if (name.size() > base.size() && name.compare(0, base.size(), base) == 0) { + FileUtil::remove((dir + "/" + name).c_str()); + } + } + closedir(handle); +} + + bool byRepThenMember(const TranslatedRow &a, const TranslatedRow &b) { if (a.repKey != b.repKey) return a.repKey < b.repKey; return a.memberKey < b.memberKey; @@ -83,7 +112,7 @@ class TextBuckets { public: TextBuckets(const std::string &prefix, unsigned int count, size_t flushBytes = 8 << 20) : prefix(prefix), count(count), flushBytes(flushBytes), closed(false) { - files.assign(count, NULL); + opened.assign(count, false); buffers.resize(count); } ~TextBuckets() { close(); } @@ -105,7 +134,6 @@ class TextBuckets { closed = true; for (unsigned int b = 0; b < count; b++) { flush(b); - if (files[b] != NULL) { fclose(files[b]); files[b] = NULL; } } } @@ -143,20 +171,32 @@ class TextBuckets { } private: + // Append-and-close per flush, as the k-mer and edge bucket writers do. Holding + // one descriptor per bucket until close needed up to 65536 of them against the + // 8192 FileUtil::fixRlimitNoFile raises to, so translation hit EMFILE at + // exactly the scale it exists for. The first flush truncates, later ones + // append. void flush(unsigned int b) { if (buffers[b].empty()) return; - if (files[b] == NULL) { - files[b] = fopen(path(prefix, b).c_str(), "wb"); - if (files[b] == NULL) { - Debug(Debug::ERROR) << "Cannot open " << path(prefix, b) << ": " << strerror(errno) - << "\n"; - EXIT(EXIT_FAILURE); - } + FILE *file = fopen(path(prefix, b).c_str(), opened[b] ? "ab" : "wb"); + if (file == NULL) { + Debug(Debug::ERROR) << "Cannot open " << path(prefix, b) << ": " << strerror(errno) + << "\n"; + EXIT(EXIT_FAILURE); } - if (fwrite(buffers[b].data(), 1, buffers[b].size(), files[b]) != buffers[b].size()) { + opened[b] = true; + if (fwrite(buffers[b].data(), 1, buffers[b].size(), file) != buffers[b].size()) { Debug(Debug::ERROR) << "Cannot write bucket " << b << "\n"; EXIT(EXIT_FAILURE); } + // Checked: on a buffered or remote filesystem this is where ENOSPC and + // quota failures first surface, and a lost record here is invisible + // downstream because whole rows keep the file self-consistent. + if (fclose(file) != 0) { + Debug(Debug::ERROR) << "Cannot close " << path(prefix, b) << ": " << strerror(errno) + << "\n"; + EXIT(EXIT_FAILURE); + } buffers[b].clear(); } @@ -164,7 +204,7 @@ class TextBuckets { unsigned int count; size_t flushBytes; std::vector buffers; - std::vector files; + std::vector opened; bool closed; }; @@ -173,7 +213,7 @@ class KeyBuckets { public: KeyBuckets(const std::string &prefix, unsigned int count, size_t bufferPairs = 1 << 16) : prefix(prefix), count(count), bufferPairs(bufferPairs), closed(false) { - files.assign(count, NULL); + opened.assign(count, false); buffers.resize(count); } ~KeyBuckets() { close(); } @@ -191,7 +231,6 @@ class KeyBuckets { closed = true; for (unsigned int b = 0; b < count; b++) { flush(b); - if (files[b] != NULL) { fclose(files[b]); files[b] = NULL; } } } @@ -216,21 +255,26 @@ class KeyBuckets { } private: + // Append-and-close per flush; see the note on TextBuckets::flush. void flush(unsigned int b) { if (buffers[b].empty()) return; - if (files[b] == NULL) { - files[b] = fopen(path(prefix, b).c_str(), "wb"); - if (files[b] == NULL) { - Debug(Debug::ERROR) << "Cannot open " << path(prefix, b) << ": " << strerror(errno) - << "\n"; - EXIT(EXIT_FAILURE); - } + FILE *file = fopen(path(prefix, b).c_str(), opened[b] ? "ab" : "wb"); + if (file == NULL) { + Debug(Debug::ERROR) << "Cannot open " << path(prefix, b) << ": " << strerror(errno) + << "\n"; + EXIT(EXIT_FAILURE); } - if (fwrite(buffers[b].data(), sizeof(KeyPair), buffers[b].size(), files[b]) != + opened[b] = true; + if (fwrite(buffers[b].data(), sizeof(KeyPair), buffers[b].size(), file) != buffers[b].size()) { Debug(Debug::ERROR) << "Cannot write bucket " << b << "\n"; EXIT(EXIT_FAILURE); } + if (fclose(file) != 0) { + Debug(Debug::ERROR) << "Cannot close " << path(prefix, b) << ": " << strerror(errno) + << "\n"; + EXIT(EXIT_FAILURE); + } buffers[b].clear(); } @@ -238,7 +282,7 @@ class KeyBuckets { unsigned int count; size_t bufferPairs; std::vector > buffers; - std::vector files; + std::vector opened; bool closed; }; @@ -397,6 +441,12 @@ int translatekeys(int argc, const char **argv, const Command &command) { const std::string tmpA = outTsv + ".bymember"; const std::string tmpB = outTsv + ".byrep"; + // Clear anything an interrupted earlier attempt left behind. A rerun only + // truncates the shards it happens to touch, and pass 2's thread/bucket + // assignment is dynamic, so shards from the previous attempt would otherwise + // survive and pass 3 would emit both attempts' rows. + removeSpillFiles(tmpA); + removeSpillFiles(tmpB); // Pass 1: bucket by member key. { diff --git a/src/test/TestKmerPartition.cpp b/src/test/TestKmerPartition.cpp index cec57e8c0..870c56f47 100644 --- a/src/test/TestKmerPartition.cpp +++ b/src/test/TestKmerPartition.cpp @@ -238,9 +238,16 @@ static void testLosslessRoundTrip(const std::string &dir) { check(sameMultiset, "grouping partition by partition sees exactly the same k-mer/sequence pairs as the whole input"); + // A legitimately empty partition is one createLayout made and no worker wrote + // to -- an *existing*, readable directory with no shards in it. A directory + // that cannot be opened is a failure, not an empty partition, because the map + // always creates the layout before the reduce runs; treating the two the same + // let a worker record an item done having silently read nothing. + const std::string emptyDir = bucketDir + "_empty"; + KmerBucketWriter::createLayout(emptyDir, 1); std::vector empty; - KmerBucketReader::readPartition(bucketDir + "_missing", 0, empty); - check(empty.empty() && KmerBucketReader::countRecords(bucketDir + "_missing", 0) == 0, + KmerBucketReader::readPartition(emptyDir, 0, empty); + check(empty.empty() && KmerBucketReader::countRecords(emptyDir, 0) == 0, "a partition nothing was written to reads back empty rather than failing"); } diff --git a/src/test/TestParallelCoordination.cpp b/src/test/TestParallelCoordination.cpp index 1ac293eb8..729b18ff2 100644 --- a/src/test/TestParallelCoordination.cpp +++ b/src/test/TestParallelCoordination.cpp @@ -129,7 +129,10 @@ static void testCounterAcrossThreads(const std::string &dir) { std::vector > perThreadValues(threadCount); std::vector threads; for (int t = 0; t < threadCount; t++) { - threads.push_back(std::thread([&counter, &perThreadValues, t, perThread]() { + // perThread is a const int and so usable without capturing it; naming it + // here trips -Wunused-lambda-capture under clang, which the aarch64 CI + // task builds with -Werror. + threads.push_back(std::thread([&counter, &perThreadValues, t]() { for (int i = 0; i < perThread; i++) { perThreadValues[t].push_back(counter.fetchAdd(1)); } From 8fd97026d8985d829645947bb5070caf09ce35df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Tue, 4 Aug 2026 13:14:42 +0000 Subject: [PATCH 21/27] More fixes. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Björn Buschkämper --- data/workflow/linclustparallel.sh | 66 ++- src/MMseqsBase.cpp | 17 +- src/commons/DenseIndex.cpp | 66 ++- src/commons/ParallelCoordination.cpp | 286 +++++++--- src/commons/ParallelCoordination.h | 58 ++- src/commons/Parameters.cpp | 20 +- src/commons/Parameters.h | 4 + src/linclust/CandidateEdge.cpp | 81 ++- src/linclust/CandidateEdge.h | 23 + src/linclust/KmerPartition.cpp | 137 ++++- src/linclust/KmerPartition.h | 7 + src/linclust/PartitionSequences.cpp | 51 +- src/linclust/PartitionSequences.h | 21 + src/linclust/alignparallel.cpp | 352 ++++++++++--- src/linclust/createrepdb.cpp | 717 +++++++++++++++++++++++--- src/linclust/greedycluster.cpp | 116 ++++- src/linclust/kmermatcherparallel.cpp | 71 ++- src/linclust/kmerreduceparallel.cpp | 157 ++++-- src/linclust/mergeclusterparallel.cpp | 45 +- src/linclust/translatecluster.cpp | 43 +- src/linclust/translatekeys.cpp | 99 +++- src/test/TestParallelCoordination.cpp | 107 ++++ src/util/createdbparallel.cpp | 257 +++++++-- src/workflow/LinclustParallel.cpp | 15 + 24 files changed, 2396 insertions(+), 420 deletions(-) diff --git a/data/workflow/linclustparallel.sh b/data/workflow/linclustparallel.sh index 83b773c21..9b8c6301a 100755 --- a/data/workflow/linclustparallel.sh +++ b/data/workflow/linclustparallel.sh @@ -47,7 +47,10 @@ [ -z "$SCRATCH_BUDGET" ] && SCRATCH_BUDGET=0 notExists() { [ ! -f "$1" ]; } -fail() { echo "Error: $1"; exit 1; } +# To stderr, not stdout: scratchUsed() runs inside a command substitution, so a +# failure message written to stdout would be captured as the measurement instead +# of being shown, and the run would die with no explanation at all. +fail() { echo "Error: $1" >&2; exit 1; } # Bytes currently sitting on the scratch filesystem for this run. # @@ -58,14 +61,46 @@ fail() { echo "Error: $1"; exit 1; } # -- it needs no per-stage accounting and stays right when the pipeline changes. # # du -sk rather than -sb: -b is GNU-only. Kibibytes are ample precision against a -# budget in terabytes. "$DB"* rather than "$DB" so the index, .index.bin, .lookup -# and headers are counted -- $DB is a prefix, not a path, and at 1e11 the index -# alone is over a terabyte of what this is meant to measure. printf "%.0f" keeps -# awk from rendering large sums in scientific notation, which --scratch-used -# would reject. +# budget in terabytes. printf "%.0f" keeps awk from rendering large sums in +# scientific notation, which --scratch-used would reject. +# +# Fails closed. This used to be one `du ... | awk ...` pipeline without pipefail: +# when du failed -- an unreadable path, a vanished directory, a stale mount -- awk +# still succeeded and printed "0B", so the caller derived its wave count as though +# the filesystem were empty and planned a run that could not fit. Reproduced with a +# missing path: exit status 0, used=0B. So du runs on its own, its status is +# checked, and a partial measurement is refused rather than rounded down to zero. +# +# $DB is a prefix, not a path, so the index, .index.bin, .lookup and headers have +# to be counted too -- at 1e11 the index alone is over a terabyte of what this +# measures. When $DB lives under $TMP (the usual case: the workflow builds it at +# $TMP/db) only $TMP is passed: GNU du de-duplicates repeated paths, but BSD/macOS +# du counts the database twice, and the surrounding code is otherwise careful to +# stay portable. scratchUsed() { - du -sk "$TMP" "$DB"* 2>/dev/null \ - | awk '{s += $1} END {printf "%.0fB\n", s * 1024}' + # The path list is built in the positional parameters rather than in a string, + # so a directory with a space in it is one argument and not two. + set -- "$TMP" + case "$DB" in + "$TMP"/*) ;; # already inside $TMP; passing it again double-counts on BSD du + *) + for _p in "$DB"*; do + if [ -e "$_p" ]; then set -- "$@" "$_p"; fi + done + ;; + esac + # `|| _du=""` rather than testing $? afterwards: under `sh -e` a command + # substitution that exits non-zero aborts the script at the assignment, so the + # check would never run. Discarding a partial measurement is the point -- + # anything less than a complete figure has to be refused, not rounded down. + _du=$(du -sk "$@" 2>/dev/null) || _du="" + if [ -z "$_du" ]; then + fail "cannot measure the scratch already in use under $TMP. +--scratch-budget is a hard ceiling on this run, and a measurement that silently +returned zero would let the k-mer shuffle be planned as though the filesystem were +empty. Check that $TMP and $DB* are readable." + fi + printf '%s\n' "$_du" | awk '{s += $1} END {printf "%.0fB\n", s * 1024}' } # Deletes an intermediate whose consumers have all finished. Nothing downstream @@ -169,7 +204,15 @@ if notExists "$INPUT.dbtype"; then # that looks complete and is not. The sentinel is written last. if notExists "$DB.coord/finalize.done"; then # shellcheck disable=SC2086 + # --write-text-index 0: no stage of this pipeline reads a text .index -- + # they all address entries through the dense .index.bin -- and generating + # it is one snprintf and one buffered write per entry, single-threaded, for + # both the sequence and the header database. Measured at 177 ns per line, + # that is ~98 h and ~36 TB at 1e12 for output nothing here consumes. Build + # the database with `mmseqs createdbparallel --write-text-index 1` if a + # stock MMseqs2 tool has to open it afterwards. $WORKER_RUNNER "$MMSEQS" createdbparallel "$INPUT" "$DB" $VERBOSITY --threads $THREADS \ + --write-text-index 0 \ || fail "createdbparallel died" fi fi @@ -219,6 +262,7 @@ dropIntermediate "$TMP/edges1" "$TMP/aln1" if notExists "$TMP/rep.keymap"; then # shellcheck disable=SC2086 "$MMSEQS" createrepdb "$DB" "$TMP/clu1.tsv" "$TMP/rep" $VERBOSITY --threads $THREADS \ + --split-memory-limit $SPLIT_MEMORY_LIMIT \ || fail "createrepdb died" fi @@ -271,8 +315,14 @@ fi # output rather than failing at the last step. if [ -f "$DB.lookup" ]; then # shellcheck disable=SC2086 + # --spill-prefix under $TMP. translatekeys derives its spill files from the + # output path when it is not told otherwise, which puts every intermediate it + # writes -- at 1e11 the first fixed-width spill alone is ~1.6 TB -- on the + # *output* filesystem, outside everything --scratch-budget accounts for. Only + # the finished file is published next to $OUT, by the rename below. "$MMSEQS" translatekeys "$TMP/clu.keys.tsv" "$DB.lookup" "$OUT.tmp" $VERBOSITY \ --threads $THREADS --split-memory-limit $SPLIT_MEMORY_LIMIT \ + --spill-prefix "$TMP/translatekeys" \ || fail "translatekeys died" mv -f "$OUT.tmp" "$OUT" else diff --git a/src/MMseqsBase.cpp b/src/MMseqsBase.cpp index 35e7b44f1..fee620085 100644 --- a/src/MMseqsBase.cpp +++ b/src/MMseqsBase.cpp @@ -311,11 +311,18 @@ std::vector baseCommands = { {"tmpDir", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::directory }}}, {"linclustparallel", linclustparallel, &par.linclustparallelworkflow, COMMAND_MAIN, "Linclust across many nodes over a shared filesystem", - "# Same clustering as linclust, computed by many independent worker\n" - "# processes that coordinate only through files: no MPI, no rank argument,\n" - "# and no node-to-node communication. Every worker of a stage runs the\n" - "# identical command line, so a stage maps onto a Slurm array job and\n" - "# workers may join late, die, or be restarted. Re-running resumes.\n\n" + "# The same clustering as linclust *for the coverage modes it supports*,\n" + "# computed by many independent worker processes that coordinate only\n" + "# through files: no MPI, no rank argument, and no node-to-node\n" + "# communication. Every worker of a stage runs the identical command line,\n" + "# so a stage maps onto a Slurm array job and workers may join late, die,\n" + "# or be restarted. Re-running resumes.\n\n" + "# Supported subset: --cov-mode 1 (target, the default here) or 2 (query).\n" + "# Stock linclust defaults to --cov-mode 0, which selects SET_COVER\n" + "# clustering and the count-table grouping rounds; neither is implemented\n" + "# here, so mode 0 is refused rather than silently answered differently.\n" + "# Nucleotide input is refused for the same reason (stock routes it to the\n" + "# v1 path, which has no counterpart here).\n\n" "# On one node\n" "mmseqs linclustparallel sequenceDB clusters.tsv tmp\n\n" "# Across 64 nodes, bounding scratch at 100 TB\n" diff --git a/src/commons/DenseIndex.cpp b/src/commons/DenseIndex.cpp index b6da0a53a..dae97ae42 100644 --- a/src/commons/DenseIndex.cpp +++ b/src/commons/DenseIndex.cpp @@ -14,8 +14,31 @@ std::string DenseIndex::fileName(const std::string &dbName) { return dbName + ".index.bin"; } +// Validated, not merely present. +// +// build() and createEmpty() used to write straight to the final path, so a worker +// killed part-way left a file that exists() accepted and every reader then failed +// on -- a zero header, a truncated entry array, or both. They now publish by +// atomic rename, but a database built by an older build (or by a killed run of +// one) can still be sitting there, so this checks what it is looking at: the +// magic, the version, and that the file is exactly as long as its own entry count +// says it should be. bool DenseIndex::exists(const std::string &dbName) { - return FileUtil::fileExists(fileName(dbName).c_str()); + const std::string path = fileName(dbName); + if (FileUtil::fileExists(path.c_str()) == false) { + return false; + } + FILE *file = fopen(path.c_str(), "rb"); + if (file == NULL) { + return false; + } + Header header; + const bool readHeader = fread(&header, sizeof(header), 1, file) == 1; + fclose(file); + if (readHeader == false || header.magic != MAGIC || header.version != VERSION) { + return false; + } + return FileUtil::getFileSize(path) == entryOffset(header.entryCount); } void DenseIndex::build(const std::string &dbName) { @@ -27,9 +50,14 @@ void DenseIndex::build(const std::string &dbName) { } const std::string outputName = fileName(dbName); - FILE *out = fopen(outputName.c_str(), "wb"); + // Written to a private sibling and renamed on success. Writing the final path + // directly meant an interrupted build left a file that exists() accepted -- + // header zeroed, because the real header is only written at the end -- and no + // restart could repair, since the path was there. + const std::string tmpName = outputName + ".tmp." + SSTR(getpid()); + FILE *out = fopen(tmpName.c_str(), "wb"); if (out == NULL) { - Debug(Debug::ERROR) << "Cannot write dense index " << outputName << ": " + Debug(Debug::ERROR) << "Cannot write dense index " << tmpName << ": " << strerror(errno) << "\n"; EXIT(EXIT_FAILURE); } @@ -110,7 +138,12 @@ void DenseIndex::build(const std::string &dbName) { EXIT(EXIT_FAILURE); } if (fclose(out) != 0) { - Debug(Debug::ERROR) << "Cannot close dense index " << outputName << "\n"; + Debug(Debug::ERROR) << "Cannot close dense index " << tmpName << "\n"; + EXIT(EXIT_FAILURE); + } + if (rename(tmpName.c_str(), outputName.c_str()) != 0) { + Debug(Debug::ERROR) << "Cannot publish dense index " << outputName << ": " + << strerror(errno) << "\n"; EXIT(EXIT_FAILURE); } @@ -185,8 +218,11 @@ DBReader::Index *DenseIndex::loadRange(const std::string &dbName, DBK size_t done = 0; while (done < count) { const size_t batch = std::min(bufferCapacity, count - done); - const long offset = static_cast(sizeof(Header) + (rowFrom + done) * sizeof(Entry)); - if (fseek(file, offset, SEEK_SET) != 0 + // fseeko/off_t, not fseek/long. A 32-bit long caps the seek at 2 GB, which + // a dense index passes at 180M entries -- three orders of magnitude below + // what this exists for -- and the overflow is silent. + const off_t offset = static_cast(sizeof(Header) + (rowFrom + done) * sizeof(Entry)); + if (fseeko(file, offset, SEEK_SET) != 0 || fread(buffer, sizeof(Entry), batch, file) != batch) { Debug(Debug::ERROR) << "Cannot read dense index " << path << " at row " << (rowFrom + done) << "\n"; @@ -262,13 +298,18 @@ void DenseIndex::writeTextIndex(const std::string &dbName) { Debug(Debug::ERROR) << "Cannot open dense index " << path << ": " << strerror(errno) << "\n"; EXIT(EXIT_FAILURE); } - if (fseek(in, static_cast(entryOffset(0)), SEEK_SET) != 0) { + if (fseeko(in, static_cast(entryOffset(0)), SEEK_SET) != 0) { Debug(Debug::ERROR) << "Cannot seek dense index " << path << "\n"; EXIT(EXIT_FAILURE); } + // Same publish-by-rename rule as build(). This one runs for hours at scale and + // is the last thing a build does, so an interrupted run leaving a half-written + // `.index` where a stock tool would read it is the likeliest way to get a + // silently short database. const std::string textIndexName = dbName + ".index"; - FILE *out = FileUtil::openAndDelete(textIndexName.c_str(), "w"); + const std::string textTmpName = textIndexName + ".tmp." + SSTR(getpid()); + FILE *out = FileUtil::openAndDelete(textTmpName.c_str(), "w"); setvbuf(out, NULL, _IOFBF, 1024 * 1024 * 4); const size_t bufferCapacity = 65536; @@ -287,7 +328,7 @@ void DenseIndex::writeTextIndex(const std::string &dbName) { (unsigned long long) buffer[i].offset, buffer[i].length); if (fwrite(line, 1, static_cast(written), out) != static_cast(written)) { - Debug(Debug::ERROR) << "Cannot write index " << textIndexName << "\n"; + Debug(Debug::ERROR) << "Cannot write index " << textTmpName << "\n"; EXIT(EXIT_FAILURE); } } @@ -296,7 +337,12 @@ void DenseIndex::writeTextIndex(const std::string &dbName) { delete[] buffer; fclose(in); if (fclose(out) != 0) { - Debug(Debug::ERROR) << "Cannot close index " << textIndexName << "\n"; + Debug(Debug::ERROR) << "Cannot close index " << textTmpName << "\n"; + EXIT(EXIT_FAILURE); + } + if (rename(textTmpName.c_str(), textIndexName.c_str()) != 0) { + Debug(Debug::ERROR) << "Cannot publish index " << textIndexName << ": " << strerror(errno) + << "\n"; EXIT(EXIT_FAILURE); } } diff --git a/src/commons/ParallelCoordination.cpp b/src/commons/ParallelCoordination.cpp index 5ee9bbd27..a4cb34adc 100644 --- a/src/commons/ParallelCoordination.cpp +++ b/src/commons/ParallelCoordination.cpp @@ -5,8 +5,11 @@ #include #include +#include #include #include +#include +#include #include #include @@ -56,23 +59,110 @@ int64_t nowSeconds() { } // namespace -FileLock::FileLock(const std::string &path) : path(path), fd(-1) { - fd = open(path.c_str(), O_RDWR | O_CREAT, 0666); - if (fd < 0) { - Debug(Debug::ERROR) << "Could not open coordination file " << path << ": " - << strerror(errno) << "\n"; - EXIT(EXIT_FAILURE); +namespace { + +// One descriptor and one mutex per path, shared by every FileLock naming it. +// +// Two fcntl properties make this necessary rather than tidy: +// +// - record locks are owned by the *process*, so two FileLock objects for one +// path in two threads both believe they hold it. Their own std::mutexes are +// different objects, so nothing serialises them and the shared state they +// guard is updated concurrently. +// - closing *any* descriptor to a file drops that process's locks on it. So a +// short-lived FileLock going out of scope silently unlocked a long-lived one +// naming the same path, mid-critical-section. +// +// Neither is hypothetical in this codebase: the stages construct SharedCounter, +// WorkQueue and bare FileLock objects over the same coordination directory, and +// createdbparallel nests a plan lock inside a queue drain. +// +// Keyed on the canonicalised *directory* plus the basename, because the lock file +// may not exist yet -- realpath() on the file itself would resolve differently +// before and after creation and hand out two entries for one file. +// +// The file is opened only when an entry is created, never speculatively. That is +// load-bearing rather than an optimisation: closing a duplicate descriptor for a +// file releases *all* of this process's record locks on it, so opening and then +// closing a second descriptor would silently drop a lock another thread was +// holding at that moment. +class LockRegistry { +public: + struct Entry { + int fd; + std::mutex mutex; + size_t refs; + Entry() : fd(-1), refs(0) {} + }; + + static LockRegistry &instance() { + static LockRegistry registry; + return registry; + } + + static std::string keyOf(const std::string &path) { + const size_t slash = path.find_last_of('/'); + const std::string dir = slash == std::string::npos ? std::string(".") + : path.substr(0, slash); + const std::string base = slash == std::string::npos ? path : path.substr(slash + 1); + char *resolved = realpath(dir.c_str(), NULL); + if (resolved == NULL) { + return path; + } + const std::string key = std::string(resolved) + "/" + base; + free(resolved); + return key; + } + + Entry *acquire(const std::string &path) { + const std::string key = keyOf(path); + std::lock_guard guard(tableMutex); + Entry *&slot = table[key]; + if (slot == NULL) { + const int fd = open(path.c_str(), O_RDWR | O_CREAT, 0666); + if (fd < 0) { + Debug(Debug::ERROR) << "Could not open coordination file " << path << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + slot = new Entry(); + slot->fd = fd; + } + slot->refs++; + return slot; } + + void release(Entry *entry) { + std::lock_guard guard(tableMutex); + if (entry == NULL || entry->refs == 0) { + return; + } + entry->refs--; + // The descriptor is deliberately *not* closed at zero. Closing it would + // drop any lock a later FileLock over the same path re-acquires, and the + // count of distinct coordination files in a run is a handful. + } + +private: + std::mutex tableMutex; + std::map table; +}; + +} // namespace + +FileLock::FileLock(const std::string &path) : path(path), fd(-1), entry(NULL) { + LockRegistry::Entry *shared = LockRegistry::instance().acquire(path); + entry = shared; + fd = shared->fd; } FileLock::~FileLock() { - if (fd >= 0) { - close(fd); - } + LockRegistry::instance().release(static_cast(entry)); + // fd belongs to the registry, not to this object; see LockRegistry::release. } void FileLock::lock() { - threadMutex.lock(); + static_cast(entry)->mutex.lock(); struct flock request; memset(&request, 0, sizeof(request)); @@ -91,7 +181,7 @@ void FileLock::lock() { // A failure here means the filesystem is not honouring fcntl locks (a // Lustre mount without -o flock is the usual cause). Continuing would // silently corrupt shared state, so stop instead. - threadMutex.unlock(); + static_cast(entry)->mutex.unlock(); EXIT(EXIT_FAILURE); } } @@ -113,7 +203,7 @@ void FileLock::unlock() { break; } - threadMutex.unlock(); + static_cast(entry)->mutex.unlock(); } SharedCounter::SharedCounter(const std::string &path) : lock(path) { @@ -202,11 +292,14 @@ void WorkQueue::initialiseLocked() { // stalled on every restart, permanently. Recounting once per open costs a // single bulk read and makes that unreachable. std::vector records; - readRecordsLocked(records); uint64_t done = 0; - for (size_t i = 0; i < records.size(); i++) { - if (records[i].state == DONE) { - done++; + for (int64_t from = 0; from < itemCount; from += SCAN_WINDOW) { + const int64_t count = std::min(SCAN_WINDOW, itemCount - from); + readRecordsLocked(from, count, records); + for (size_t i = 0; i < records.size(); i++) { + if (records[i].state == DONE) { + done++; + } } } if (done != header.doneCount) { @@ -284,15 +377,20 @@ WorkQueue::Record WorkQueue::readRecordLocked(int64_t index) { return record; } -void WorkQueue::readRecordsLocked(std::vector &out) { - out.resize(static_cast(itemCount)); - if (itemCount == 0) { +void WorkQueue::readRecordsLocked(int64_t from, int64_t count, std::vector &out) { + if (count < 0 || from < 0 || from + count > itemCount) { + Debug(Debug::ERROR) << "Work queue " << path << " asked for records [" << from << ", " + << (from + count) << ") of " << itemCount << "\n"; + EXIT(EXIT_FAILURE); + } + out.resize(static_cast(count)); + if (count == 0) { return; } - const size_t bytes = static_cast(itemCount) * sizeof(Record); - if (preadFully(lock.getFd(), &out[0], bytes, recordOffset(0)) != static_cast(bytes)) { - Debug(Debug::ERROR) << "Could not read the " << itemCount << " records of work queue " - << path << ": " << strerror(errno) << "\n"; + const size_t bytes = static_cast(count) * sizeof(Record); + if (preadFully(lock.getFd(), &out[0], bytes, recordOffset(from)) != static_cast(bytes)) { + Debug(Debug::ERROR) << "Could not read records [" << from << ", " << (from + count) + << ") of work queue " << path << ": " << strerror(errno) << "\n"; EXIT(EXIT_FAILURE); } } @@ -311,46 +409,60 @@ int64_t WorkQueue::claim(int64_t workerId, int64_t leaseSeconds) { Header header = readHeaderLocked(); const int64_t now = nowSeconds(); + // Scanned in bounded windows from nextHint rather than reading the whole + // record array. Both parts matter: starting at nextHint skips the finished + // prefix, and stopping after SCAN_WINDOW records bounds the read even when + // the prefix has not advanced -- which is the case while the queue is being + // worked, since nextHint only moves past items that are *DONE*. std::vector records; - readRecordsLocked(records); - int64_t firstUnfinished = -1; - for (int64_t index = static_cast(header.nextHint); index < itemCount; index++) { - Record record = records[static_cast(index)]; - if (record.state == DONE) { - continue; - } - if (firstUnfinished < 0) { - firstUnfinished = index; - } - // Untouched, or held by a worker whose lease ran out -- in both cases the - // item is ours to take. Re-claiming after a lease expiry is what makes a - // killed job recoverable without an operator deciding what to redo. - if (record.state == PENDING - || (record.state == CLAIMED && static_cast(record.leaseExpiry) <= now)) { - record.state = CLAIMED; - record.worker = static_cast(workerId); - record.leaseExpiry = static_cast(now + leaseSeconds); - writeRecordLocked(index, record); - - if (firstUnfinished > static_cast(header.nextHint)) { - header.nextHint = static_cast(firstUnfinished); - writeHeaderLocked(header); + int64_t claimed = -1; + for (int64_t base = static_cast(header.nextHint); base < itemCount && claimed < 0; + base += SCAN_WINDOW) { + const int64_t count = std::min(SCAN_WINDOW, itemCount - base); + readRecordsLocked(base, count, records); + for (int64_t offset = 0; offset < count; offset++) { + const int64_t index = base + offset; + Record record = records[static_cast(offset)]; + if (record.state == DONE) { + continue; + } + if (firstUnfinished < 0) { + firstUnfinished = index; + } + // Untouched, or held by a worker whose lease ran out -- in both cases + // the item is ours to take. Re-claiming after a lease expiry is what + // makes a killed job recoverable without an operator deciding what to + // redo. + if (record.state == PENDING + || (record.state == CLAIMED + && static_cast(record.leaseExpiry) <= now)) { + record.state = CLAIMED; + record.worker = static_cast(workerId); + record.leaseExpiry = static_cast(now + leaseSeconds); + writeRecordLocked(index, record); + claimed = index; + break; } - fsync(lock.getFd()); - lock.unlock(); - return index; } } if (firstUnfinished > static_cast(header.nextHint)) { header.nextHint = static_cast(firstUnfinished); writeHeaderLocked(header); - fsync(lock.getFd()); } + // Kept, against the reviewers' suggestion to drop it. It is not durability + // against a crash -- the stage is re-run for that -- it is *visibility*: the + // record was pwritten into the page cache, and a worker on another node that + // takes the lock next must see it or it will claim the same item. fcntl + // lock/unlock does imply a cache flush on a conformant NFSv4 client, but the + // cost of that assumption being wrong is two workers silently running one item. + // The O(itemCount) read this used to do alongside it was the real cost, and + // that is what the windowed scan above removes. + fsync(lock.getFd()); lock.unlock(); - return -1; + return claimed; } void WorkQueue::completeLocked(int64_t index, int64_t workerId) { @@ -362,6 +474,20 @@ void WorkQueue::completeLocked(int64_t index, int64_t workerId) { // finished before it died. return; } + if (record.state == CLAIMED && record.worker != static_cast(workerId)) { + // Someone else holds it. Our lease lapsed -- a stalled node, a long GC of + // the filesystem, a clock skew -- and another worker re-claimed the item + // and is running it *now*. Recording DONE here would tell the stage an + // in-progress item is finished, and for the reduce it would also make this + // worker the authority for that partition's edge blocks while the worker + // actually producing them is not. + // + // renew() and release() have always guarded on ownership; only complete() + // did not. Returning silently is right: our output for the item is + // superseded by the holder's, which is exactly what the EdgeBlockHeader + // filter is built to express. + return; + } record.state = DONE; record.worker = static_cast(workerId); @@ -422,13 +548,24 @@ int64_t WorkQueue::getDoneCount() { bool WorkQueue::hasLiveClaim() { const int64_t now = nowSeconds(); lock.lock(); + // From nextHint, in windows, stopping at the first live claim. Every idle + // worker calls this every pollSeconds at the tail of a stage, so reading the + // whole array here was the second O(itemCount) read under the global lock -- + // and the one that most directly crowded out the heartbeat renewals live + // workers depend on. + const Header header = readHeaderLocked(); std::vector records; - readRecordsLocked(records); bool live = false; - for (int64_t i = 0; i < itemCount && live == false; i++) { - const Record &record = records[static_cast(i)]; - if (record.state == CLAIMED && static_cast(record.leaseExpiry) > now) { - live = true; + for (int64_t base = static_cast(header.nextHint); base < itemCount && live == false; + base += SCAN_WINDOW) { + const int64_t count = std::min(SCAN_WINDOW, itemCount - base); + readRecordsLocked(base, count, records); + for (int64_t i = 0; i < count; i++) { + const Record &record = records[static_cast(i)]; + if (record.state == CLAIMED && static_cast(record.leaseExpiry) > now) { + live = true; + break; + } } } lock.unlock(); @@ -457,16 +594,27 @@ bool WorkQueue::readCompletedWorkers(const std::string &path, std::vector(header.itemCount), -1); - for (uint64_t i = 0; i < header.itemCount; i++) { - Record record; - if (preadFully(fd, &record, sizeof(Record), recordOffset(static_cast(i))) != - static_cast(sizeof(Record))) { + // Read in blocks, not one 16-byte pread per item: the align stage calls this + // once per reduce wave over a queue that has up to 1e6 items at the 1e12 + // sizing, and a pread each was 1e6 round trips before any edge was read. + std::vector block; + for (uint64_t from = 0; from < header.itemCount; from += static_cast(SCAN_WINDOW)) { + const uint64_t count = + std::min(static_cast(SCAN_WINDOW), header.itemCount - from); + block.resize(static_cast(count)); + const size_t bytes = static_cast(count) * sizeof(Record); + if (preadFully(fd, &block[0], bytes, recordOffset(static_cast(from))) + != static_cast(bytes)) { close(fd); - Debug(Debug::ERROR) << "Cannot read record " << i << " of work queue " << path << "\n"; + Debug(Debug::ERROR) << "Cannot read records [" << from << ", " << (from + count) + << ") of work queue " << path << "\n"; EXIT(EXIT_FAILURE); } - if (record.state == DONE) { - workers[static_cast(i)] = static_cast(record.worker); + for (uint64_t i = 0; i < count; i++) { + if (block[static_cast(i)].state == DONE) { + workers[static_cast(from + i)] = + static_cast(block[static_cast(i)].worker); + } } } close(fd); @@ -476,6 +624,12 @@ bool WorkQueue::readCompletedWorkers(const std::string &path, std::vector= itemCount) { @@ -484,6 +638,7 @@ bool WorkQueue::awaitAll(unsigned int pollSeconds, unsigned int stallSeconds) { if (done != lastDone) { lastDone = done; lastProgress = nowSeconds(); + wait = pollSeconds; } else if (stallSeconds > 0 && nowSeconds() - lastProgress > static_cast(stallSeconds)) { Debug(Debug::WARNING) << "Work queue " << path << " made no progress for " @@ -491,6 +646,9 @@ bool WorkQueue::awaitAll(unsigned int pollSeconds, unsigned int stallSeconds) { << " done)\n"; return false; } - sleep(pollSeconds); + sleep(wait); + if (wait < maxWait) { + wait = std::min(maxWait, wait * 2); + } } } diff --git a/src/commons/ParallelCoordination.h b/src/commons/ParallelCoordination.h index 0aa067d43..b9e428e14 100644 --- a/src/commons/ParallelCoordination.h +++ b/src/commons/ParallelCoordination.h @@ -1,6 +1,7 @@ #ifndef MMSEQS_PARALLELCOORDINATION_H #define MMSEQS_PARALLELCOORDINATION_H +#include #include #include #include @@ -25,13 +26,19 @@ // - locks are owned by the *process*, not the thread, so a process-local mutex // is needed as well to serialise threads within one worker; // - locks are dropped when *any* file descriptor to the file is closed, so the -// descriptor is opened once and owned for the object's lifetime. +// descriptor must be owned in exactly one place. // Locks are released automatically when a process dies, so a crashed worker can // never deadlock the run. +// +// Both consequences are per *file*, not per object, so the descriptor and the +// mutex live in a process-global registry keyed on the canonical path (see +// LockRegistry in the .cpp). Two FileLock instances naming one path therefore +// serialise against each other, and neither can close the descriptor out from +// under the other. class FileLock { public: - // Opens (creating if needed) the lock file and keeps the descriptor for the - // object's lifetime. Does not acquire the lock. + // Joins (creating the file if needed) the process-wide lock state for this + // path. Does not acquire the lock. explicit FileLock(const std::string &path); ~FileLock(); @@ -48,8 +55,9 @@ class FileLock { FileLock &operator=(const FileLock &); std::string path; + // Both borrowed from the per-path registry entry, which outlives this object. int fd; - std::mutex threadMutex; + void *entry; }; // A single 64-bit integer in a shared file, updated atomically across nodes. @@ -170,9 +178,20 @@ class WorkQueue { unsigned int stallSeconds = 2 * DEFAULT_LEASE_SECONDS) { int64_t lastDone = -1; int64_t lastProgress = static_cast(time(NULL)); + // Idle polling backs off. At the tail of a stage every worker that has run + // out of claimable items sits in the branch below, and each pass costs a + // claim() and a hasLiveClaim() through the one global lock -- thousands of + // workers hammering the lock exactly while the last few holders need it for + // their heartbeats. Doubling up to a minute keeps a late joiner responsive + // (the first waits are still pollSeconds) without that pile-up. Reset on + // every observed completion, so a queue that starts moving again is picked + // up promptly. + unsigned int wait = pollSeconds; + const unsigned int maxWait = 60; while (true) { const int64_t item = claim(workerId); if (item >= 0) { + wait = pollSeconds; // Heartbeat for the duration of the item. Without it any item // taking longer than the lease -- which at 1e11 is every reduce // partition and every align bucket, both hours of work -- looks @@ -207,6 +226,7 @@ class WorkQueue { if (done != lastDone) { lastDone = done; lastProgress = static_cast(time(NULL)); + wait = pollSeconds; } else if (hasLiveClaim()) { // Someone is still working, and heartbeating to say so. A stage // whose last item takes hours must not be declared stalled. @@ -216,7 +236,10 @@ class WorkQueue { static_cast(stallSeconds)) { return false; } - sleep(pollSeconds); + sleep(wait); + if (wait < maxWait) { + wait = std::min(maxWait, wait * 2); + } } } @@ -251,6 +274,13 @@ class WorkQueue { uint64_t leaseExpiry; }; + // The comment above has always said the sizes are asserted; they were not. + // A queue is written by one worker and read by others that may have been + // built at a different time, so a layout change has to fail the build rather + // than reinterpret an existing file. + static_assert(sizeof(Header) == 64, "WorkQueue::Header is written to disk verbatim"); + static_assert(sizeof(Record) == 16, "WorkQueue::Record is written to disk verbatim"); + static const uint64_t MAGIC = 0x4d4d51554555453fULL; // "MMQUEUE?" static const uint64_t VERSION = 1; @@ -258,15 +288,27 @@ class WorkQueue { Header readHeaderLocked(); void writeHeaderLocked(const Header &header); Record readRecordLocked(int64_t index); - // The whole record array in one read, for the scans that would otherwise issue - // itemCount separate 16-byte preads while holding the lock. On a shared + // A contiguous run of records in one read, for the scans that would otherwise + // issue one 16-byte pread per item while holding the lock. On a shared // filesystem each of those is a cross-node round trip, and at P = 8192 with // every idle worker polling every 5 s they crowd out the heartbeat renewals // that keep live workers' leases from lapsing. - void readRecordsLocked(std::vector &out); + // + // out is indexed *relative to `from`*. Reading [from, from + count) rather + // than the whole array is what keeps claim() O(1) in the item count: this used + // to pread from record 0 every time, so a 1,048,576-item queue -- the map's + // size at 1e12 -- moved 16 MB under the global lock per claim, measured at + // 13.2 ms against 4.2 ms for a 4096-item queue. + void readRecordsLocked(int64_t from, int64_t count, std::vector &out); void writeRecordLocked(int64_t index, const Record &record); void completeLocked(int64_t index, int64_t workerId); + // Records read per pass when scanning for a claimable item. Bounds the read + // under the lock; the scan extends by another window only when the whole + // window is DONE or live-leased, which happens on a queue whose head is a long + // finished prefix -- and nextHint then skips it on the next call. + static const int64_t SCAN_WINDOW = 4096; + static size_t recordOffset(int64_t index) { return sizeof(Header) + static_cast(index) * sizeof(Record); } diff --git a/src/commons/Parameters.cpp b/src/commons/Parameters.cpp index d6f32fcc2..d9f4e614b 100644 --- a/src/commons/Parameters.cpp +++ b/src/commons/Parameters.cpp @@ -62,6 +62,7 @@ Parameters::Parameters(): PARAM_KEY_MAP(PARAM_KEY_MAP_ID, "--key-map", "Sub-key map", "Maps this database's dense sub-keys back to the original keys, as written by createrepdb. Needed with --filter-cludb-file when the pass runs on a re-keyed representative database.", typeid(std::string), (void *) &keyMapFile, "", MMseqsParameter::COMMAND_ALIGN | MMseqsParameter::COMMAND_EXPERT), PARAM_SCRATCH_BUDGET(PARAM_SCRATCH_BUDGET_ID, "--scratch-budget", "Scratch budget", "Total scratch the run may occupy. The k-mer extraction wave count and the partition count are derived from this together with --split-memory-limit, rather than set by hand. Default (0) for a single wave. E.g. 100T, 500T", typeid(ByteParser), (void *) &scratchBudget, "^(0|[1-9]{1}[0-9]*(B|K|M|G|T)?)$", MMseqsParameter::COMMAND_EXPERT), PARAM_SCRATCH_USED(PARAM_SCRATCH_USED_ID, "--scratch-used", "Scratch already used", "Bytes of --scratch-budget already occupied when this stage starts. The workflow measures it, so a later pass accounts for what earlier passes left on disk. 0 derives it from the input database alone.", typeid(ByteParser), (void *) &scratchUsed, "^(0|[1-9]{1}[0-9]*(B|K|M|G|T)?)$", MMseqsParameter::COMMAND_EXPERT), + PARAM_SPILL_PREFIX(PARAM_SPILL_PREFIX_ID, "--spill-prefix", "Spill prefix", "Path prefix for this command's temporary spill files. Default (empty) puts them beside the output, which is outside the scratch filesystem --scratch-budget accounts for. Point it inside the run's tmp directory so the spill is counted and cleaned up with the rest of the run.", typeid(std::string), (void *) &spillPrefix, "", MMseqsParameter::COMMAND_EXPERT), PARAM_DISK_SPACE_LIMIT(PARAM_DISK_SPACE_LIMIT_ID, "--disk-space-limit", "Disk space limit", "Set max disk space to use for reverse profile searches. E.g. 800B, 5K, 10M, 1G. Default (0) to all available disk space in the temp folder", typeid(ByteParser), (void *) &diskSpaceLimit, "^(0|[1-9]{1}[0-9]*(B|K|M|G|T)?)$", MMseqsParameter::COMMAND_COMMON | MMseqsParameter::COMMAND_PREFILTER | MMseqsParameter::COMMAND_EXPERT), PARAM_SPLIT_AMINOACID(PARAM_SPLIT_AMINOACID_ID, "--split-aa", "Split by amino acid", "Try to find the best split boundaries by entry lengths", typeid(bool), (void *) &splitAA, "$", MMseqsParameter::COMMAND_EXPERT), PARAM_SUB_MAT(PARAM_SUB_MAT_ID, "--sub-mat", "Substitution matrix", "Substitution matrix file", typeid(MultiParam>), (void *) &scoringMatrixFile, "", MMseqsParameter::COMMAND_COMMON | MMseqsParameter::COMMAND_EXPERT), @@ -227,6 +228,7 @@ Parameters::Parameters(): PARAM_CREATEDB_MODE(PARAM_CREATEDB_MODE_ID, "--createdb-mode", "Createdb mode", "Createdb mode 0: copy data, 1: soft link data and write new index (works only with single line fasta/q) 2: GPU compatible db", typeid(int), (void *) &createdbMode, "^[0-2]{1}$"), PARAM_SHUFFLE(PARAM_SHUFFLE_ID, "--shuffle", "Shuffle input database", "Shuffle input database", typeid(bool), (void *) &shuffleDatabase, ""), PARAM_WRITE_LOOKUP(PARAM_WRITE_LOOKUP_ID, "--write-lookup", "Write lookup file", "write .lookup file containing mapping from internal id, fasta id and file number", typeid(int), (void *) &writeLookup, "^[0-1]{1}", MMseqsParameter::COMMAND_EXPERT), + PARAM_WRITE_TEXT_INDEX(PARAM_WRITE_TEXT_INDEX_ID, "--write-text-index", "Write text index", "Also write the stock-compatible text .index alongside the dense .index.bin. None of the distributed stages read it, and at 1e12 sequences it is ~36 TB and ~98 h of single-threaded formatting; turn it off unless a stock MMseqs2 tool has to open the database.", typeid(int), (void *) &writeTextIndex, "^[0-1]{1}$", MMseqsParameter::COMMAND_EXPERT), PARAM_USE_HEADER_FILE(PARAM_USE_HEADER_FILE_ID, "--use-header-file", "Use header DB", "use the sequence header DB instead of the body to map the entry keys", typeid(bool), (void *) &useHeaderFile, ""), // setextendeddbtype PARAM_EXTENDED_DBTYPE(PARAM_EXTENDED_DBTYPE_ID, "--extended-dbtype", "Extended dbtype", "Set extended dbtype 1: compressed, 2: need src, 4: context pseudoe cnts", typeid(int), (void *) &extendedDbtype, "^[0-4]{1}"), @@ -921,8 +923,18 @@ Parameters::Parameters(): createdbparallel.push_back(&PARAM_WRITE_LOOKUP); createdbparallel.push_back(&PARAM_DB_TYPE); createdbparallel.push_back(&PARAM_CHUNK_SIZE); + createdbparallel.push_back(&PARAM_WRITE_TEXT_INDEX); createdbparallel.push_back(&PARAM_THREADS); - createdbparallel.push_back(&PARAM_COMPRESSED); + // No PARAM_COMPRESSED. emitChunk pwrites raw sequence and header bytes into + // preallocated files at offsets a plan derived from *uncompressed* lengths, + // so there is nowhere for compression to happen -- but finalize passed + // par.compressed straight into DBWriter::writeDbtypeFile, marking those raw + // files as compressed. Readers then decoded raw bytes as compressed records: + // `createdbparallel in.fasta db --compressed 1` followed by + // `convert2fasta db out.fasta` segfaulted. Supporting it needs a + // compression-aware planner, since compressed offsets cannot be derived from + // sequence lengths; until then the flag is not offered rather than accepted + // and ignored. createdbparallel.push_back(&PARAM_V); // kmermatcherparallel @@ -1001,6 +1013,9 @@ Parameters::Parameters(): mergeclusterparallel.push_back(&PARAM_V); // createrepdb + // --split-memory-limit sizes the bucketed pass that builds the pass-2 filter + // gate; the copy itself is bounded by its chunk size, not by this. + createrepdb.push_back(&PARAM_SPLIT_MEMORY_LIMIT); createrepdb.push_back(&PARAM_THREADS); createrepdb.push_back(&PARAM_V); @@ -1010,6 +1025,7 @@ Parameters::Parameters(): // translatekeys translatekeys.push_back(&PARAM_SPLIT_MEMORY_LIMIT); + translatekeys.push_back(&PARAM_SPILL_PREFIX); translatekeys.push_back(&PARAM_THREADS); translatekeys.push_back(&PARAM_V); @@ -2636,6 +2652,8 @@ void Parameters::setDefaults() { keyMapFile = ""; scratchBudget = 0; scratchUsed = 0; + spillPrefix = ""; + writeTextIndex = 1; diskSpaceLimit = 0; splitAA = false; spacedKmerPattern = ""; diff --git a/src/commons/Parameters.h b/src/commons/Parameters.h index 140511709..940a7725c 100644 --- a/src/commons/Parameters.h +++ b/src/commons/Parameters.h @@ -444,6 +444,8 @@ class Parameters { int kmerWave; // kmermatcherparallel: which extraction wave to write size_t scratchBudget; size_t scratchUsed; // Bytes of the budget already occupied when a stage starts + std::string spillPrefix; // translatekeys: where its spill files go, when not beside the output + int writeTextIndex; // createdbparallel: also write the stock-compatible text .index size_t diskSpaceLimit; // Maximum disk space in bytes for sliced reverse profile search bool splitAA; // Split database by amino acid count instead int preloadMode; // Preload mode of database @@ -841,6 +843,7 @@ class Parameters { PARAMETER(PARAM_KEY_MAP) PARAMETER(PARAM_SCRATCH_BUDGET) PARAMETER(PARAM_SCRATCH_USED) + PARAMETER(PARAM_SPILL_PREFIX) PARAMETER(PARAM_DISK_SPACE_LIMIT) PARAMETER(PARAM_SPLIT_AMINOACID) PARAMETER(PARAM_SUB_MAT) @@ -1026,6 +1029,7 @@ class Parameters { PARAMETER(PARAM_CREATEDB_MODE) PARAMETER(PARAM_SHUFFLE) PARAMETER(PARAM_WRITE_LOOKUP) + PARAMETER(PARAM_WRITE_TEXT_INDEX) // convert2fasta PARAMETER(PARAM_USE_HEADER_FILE) diff --git a/src/linclust/CandidateEdge.cpp b/src/linclust/CandidateEdge.cpp index a7c4ff27e..937198c26 100644 --- a/src/linclust/CandidateEdge.cpp +++ b/src/linclust/CandidateEdge.cpp @@ -11,8 +11,25 @@ #include #include +#include #include +namespace { +// See deriveDescriptorBudget in KmerPartition.cpp: bucket descriptors are kept +// open across flushes up to a bounded set, so the common case (a bucket count +// well inside the descriptor limit) costs one open per bucket per stage rather +// than one per flush, while a 65536-bucket run still cannot exhaust descriptors. +size_t deriveEdgeDescriptorBudget(unsigned int bucketCount) { + size_t allowed = 256; + struct rlimit limit; + if (getrlimit(RLIMIT_NOFILE, &limit) == 0 && limit.rlim_cur != RLIM_INFINITY + && limit.rlim_cur > 64) { + allowed = static_cast(limit.rlim_cur) / 2; + } + return std::min(std::max(allowed, 16), std::max(bucketCount, 1)); +} +} // namespace + std::string EdgeWriter::partitionPath(const std::string &dir, unsigned int partition) { return dir + "/p" + SSTR(partition) + ".edges"; } @@ -142,7 +159,14 @@ EdgeBucketWriter::EdgeBucketWriter(const std::string &dir, unsigned int bucketCo const size_t perBucket = bufferBudgetBytes / (bucketCount * sizeof(CandidateEdge)); edgesPerBuffer = std::max(perBucket, 64); buffers.resize(bucketCount); + // Reserved up front, so a std::vector's geometric growth cannot leave the + // buffers holding twice the budget once they settle at edgesPerBuffer. + for (unsigned int b = 0; b < bucketCount; b++) { + buffers[b].reserve(edgesPerBuffer); + } files.assign(bucketCount, NULL); + descriptorBudget = deriveEdgeDescriptorBudget(bucketCount); + openFiles = 0; } EdgeBucketWriter::~EdgeBucketWriter() { @@ -162,14 +186,13 @@ void EdgeBucketWriter::flush(unsigned int bucket) { // Opened lazily: a worker whose partitions produced nothing for a bucket // should not cost a descriptor or an empty file. const std::string path = shardPath(bucket); - // Append-and-close per flush, for the same reason as KmerBucketWriter: - // one descriptor per bucket would need up to 65536 of them. files[bucket] = fopen(path.c_str(), "ab"); if (files[bucket] == NULL) { Debug(Debug::ERROR) << "Cannot open edge bucket " << path << ": " << strerror(errno) << "\n"; EXIT(EXIT_FAILURE); } + openFiles++; } // Header then records, as one write each. The header names the producer so a // crashed worker's superseded copy can be told apart from a second partition @@ -196,17 +219,32 @@ void EdgeBucketWriter::flush(unsigned int bucket) { << " of " << dir << ": " << strerror(errno) << "\n"; EXIT(EXIT_FAILURE); } - // Closed here, not held until close(): with up to 65536 buckets against the - // 8192 descriptors fixRlimitNoFile raises to, keeping one open per bucket runs - // the process out of descriptors partway through the reduce. This is what the - // comment above the open has always claimed; only the fclose was missing. - if (fclose(files[bucket]) != 0) { + // Held open while the descriptor budget allows, and closed past it. With up to + // 65536 buckets against the 8192 descriptors fixRlimitNoFile raises to, keeping + // one open per bucket unconditionally would run the process out of them + // partway through the reduce; closing on every flush instead made the stage + // issue an open and a close per buffer, which is what a metadata server + // notices at 1e12. + if (openFiles > descriptorBudget) { + closeFile(bucket); + } + buffer.clear(); +} + +void EdgeBucketWriter::closeFile(unsigned int bucket) { + if (files[bucket] == NULL) { + return; + } + FILE *file = files[bucket]; + files[bucket] = NULL; + openFiles--; + // Checked: a buffered or remote filesystem reports ENOSPC and quota failures + // here, and the queue marks the partition done right after flushAll(). + if (fclose(file) != 0) { Debug(Debug::ERROR) << "Cannot close edge bucket " << bucket << " of " << dir << ": " << strerror(errno) << "\n"; EXIT(EXIT_FAILURE); } - files[bucket] = NULL; - buffer.clear(); } void EdgeBucketWriter::beginPartition(unsigned int partition, int64_t worker) { @@ -239,6 +277,14 @@ void EdgeBucketWriter::append(unsigned int bucket, const CandidateEdge &edge) { void EdgeBucketWriter::flushAll() { for (unsigned int b = 0; b < bucketCount; b++) { flush(b); + // Descriptors that survive a flush hold the block in a stdio buffer, so + // getting it to the OS -- what this call owes its caller before the item + // is marked done -- now takes an explicit, checked fflush. + if (files[b] != NULL && fflush(files[b]) != 0) { + Debug(Debug::ERROR) << "Cannot flush edge bucket " << b << " of " << dir << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } } } @@ -249,6 +295,7 @@ void EdgeBucketWriter::close() { closed = true; for (unsigned int b = 0; b < bucketCount; b++) { flush(b); + closeFile(b); } } @@ -322,9 +369,21 @@ size_t EdgeBucketReader::readShard(const std::string &path, const std::vector= authority.size()) { + Debug(Debug::ERROR) << "Edge shard " << path << " holds a block from partition " + << header.partition << ", but the reduce covered only " + << authority.size() << " partitions. The edge directory holds " + << "output from a run with a different partitioning.\n"; + EXIT(EXIT_FAILURE); + } const bool wanted = - authority.empty() || header.partition >= authority.size() || - authority[header.partition] == static_cast(header.worker); + authority.empty() || authority[header.partition] == static_cast(header.worker); if (wanted == false) { // A superseded copy: this worker did not record the partition done, so // another redid it and its edges are the ones that count. diff --git a/src/linclust/CandidateEdge.h b/src/linclust/CandidateEdge.h index ba63a076e..9f0cd2126 100644 --- a/src/linclust/CandidateEdge.h +++ b/src/linclust/CandidateEdge.h @@ -18,6 +18,23 @@ struct __attribute__((__packed__)) CandidateEdge { // bytes cheaper per side than a full 64-bit key. uint8_t repBytes[6]; uint8_t memberBytes[6]; + // 16 bits deliberately, and *not* wide enough to hold every diagonal a 65535 + // residue sequence can produce. That is stock's convention, not an oversight: + // + // - stock stores it in `short KmerEntry::diagonal` (kmermatcher.h:187,199) + // and truncates an int into it at kmermatcher.cpp:2050, exactly as + // collectRoundEdges does here; + // - the prefilter hands it on as `unsigned short hit_t::diagonal` + // (QueryMatcher.h:36); + // - the aligner *undoes* the truncation. ungappedAlign takes an + // `unsigned short` and DistanceCalculator::computeUngappedAlignment + // (DistanceCalculator.h:93-112) tries every real diagonal congruent to it + // mod 65536 that the two lengths allow, keeping the best-scoring one. + // + // So a diagonal of 45000 stored as -20536 is recovered by the aligner, and two + // real diagonals that alias mod 65536 alias in stock too. Widening this would + // diverge from stock rather than converge on it, and cost 6% of the largest + // intermediate the pipeline writes. int16_t diagonal; // Nucleotide strand. Stock carries it in bit 63 of the representative key, // which a 48-bit key has no room for. @@ -172,6 +189,7 @@ class EdgeBucketWriter { EdgeBucketWriter &operator=(const EdgeBucketWriter &); void flush(unsigned int bucket); + void closeFile(unsigned int bucket); std::string shardPath(unsigned int bucket) const; std::string dir; @@ -180,6 +198,11 @@ class EdgeBucketWriter { size_t edgesPerBuffer; std::vector > buffers; std::vector files; + // Descriptors kept open across flushes, up to descriptorBudget. Plain counters: + // one writer belongs to one process and its appends come from the single + // thread that drains the work queue. + size_t descriptorBudget; + size_t openFiles; uint64_t edgeCount; bool closed; unsigned int currentPartition; diff --git a/src/linclust/KmerPartition.cpp b/src/linclust/KmerPartition.cpp index af05091f3..f9a244b67 100644 --- a/src/linclust/KmerPartition.cpp +++ b/src/linclust/KmerPartition.cpp @@ -9,6 +9,7 @@ #include #include +#include #include KmerPartitioner::KmerPartitioner(unsigned int partitionCount) : partitionCount(partitionCount) { @@ -40,6 +41,30 @@ uint64_t divideRoundingUp(uint64_t value, uint64_t divisor) { return (value + divisor - 1) / divisor; } +// How many bucket descriptors one writer may hold open across flushes. +// +// The reason the writers used to open, write and close on *every* flush is real: +// P can reach 65536 against the 8192 soft limit FileUtil::fixRlimitNoFile raises +// to, so one descriptor per bucket runs the process out of them partway through. +// But that made the map issue ~1e9 open+close pairs at 1e12 against a single +// metadata server -- 1e6 work items x 1024 partitions, plus intra-item flushes. +// +// Keeping a bounded set open costs nothing and removes almost all of it: a wave +// only writes P/W partitions, so the common case fits inside the budget entirely +// and every flush becomes a plain fwrite. Buckets past the budget fall back to +// open-write-close, which is exactly today's cost for those alone. +size_t deriveDescriptorBudget(unsigned int activeCount) { + // Half the soft limit, so the rest of the process -- the sequence database, + // the coordination files, the work queue -- keeps its share. + size_t allowed = 256; + struct rlimit limit; + if (getrlimit(RLIMIT_NOFILE, &limit) == 0 && limit.rlim_cur != RLIM_INFINITY + && limit.rlim_cur > 64) { + allowed = static_cast(limit.rlim_cur) / 2; + } + return std::min(std::max(allowed, 16), std::max(activeCount, 1)); +} + unsigned int roundUpToPowerOfTwo(uint64_t value) { // Above 2^31 the shift below wraps to 0 and the loop never terminates, which // an extreme --split-memory-limit or --scratch-budget can reach. Nothing here @@ -180,17 +205,53 @@ KmerBucketWriter::KmerBucketWriter(const std::string &dir, unsigned int partitio // At least a handful of records per partition even with a tiny budget, so a // large partition count degrades to more frequent flushes rather than to // one write syscall per k-mer. - const size_t perPartition = bufferBudgetBytes / (partitionCount * sizeof(KmerRecord)); + // + // Divided by the partitions this wave actually writes, not by P. A wave owns + // the slice [partitionFrom, partitionTo) and drops every append outside it, so + // dividing by P gave each live partition a buffer waveCount times smaller than + // the budget allowed -- and the write size is exactly what a parallel + // filesystem cares about. + const unsigned int activeCount = + (this->partitionTo > partitionFrom) ? (this->partitionTo - partitionFrom) : 1; + const size_t perPartition = bufferBudgetBytes / (activeCount * sizeof(KmerRecord)); recordsPerBuffer = std::max(perPartition, 16); buffers.resize(partitionCount); + // Reserved up front: push_back until size() >= recordsPerBuffer lets a + // std::vector's geometric growth land on twice the budgeted capacity, so a + // 1 GB budget became 2 GB resident across the active partitions. + for (unsigned int p = partitionFrom; p < this->partitionTo && p < partitionCount; p++) { + buffers[p].reserve(recordsPerBuffer); + } files.assign(partitionCount, NULL); recordCounts.assign(partitionCount, 0); + descriptorBudget = deriveDescriptorBudget(activeCount); + openFiles.store(0); } KmerBucketWriter::~KmerBucketWriter() { close(); } +// Caller must hold mutexes[partition]. +void KmerBucketWriter::closeFile(unsigned int partition) { + if (files[partition] == NULL) { + return; + } + // Checked: buffered I/O can report ENOSPC, a quota, or a remote filesystem + // error only at close, and the queue marks the item done straight after + // flushAll(). An unchecked close here let an item be recorded complete with + // k-mers missing, which the reduce then reads as "this k-mer had no partner" + // -- a wrong clustering with no diagnostic. + FILE *file = files[partition]; + files[partition] = NULL; + openFiles.fetch_sub(1); + if (fclose(file) != 0) { + Debug(Debug::ERROR) << "Cannot close k-mer bucket " << partition << " of " << dir << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } +} + void KmerBucketWriter::flush(unsigned int partition) { std::vector &buffer = buffers[partition]; if (buffer.empty()) { @@ -199,18 +260,14 @@ void KmerBucketWriter::flush(unsigned int partition) { if (files[partition] == NULL) { // Opened lazily: with 8192 partitions and a sparse shard, most buckets // stay untouched and should not cost a file descriptor or an empty file. - // Opened in append mode and closed again after the write (see below). - // Holding one descriptor per partition for the life of the writer needs P - // of them -- 8192 at the 1e12 sizing -- against a soft limit - // FileUtil::fixRlimitNoFile only raises to 8192, so the last partitions - // would fail to open. Append is safe because this shard belongs to this - // worker alone. + // Append is safe because this shard belongs to this worker alone. const std::string path = partitionDir(dir, partition) + "/" + shardId + ".kmers"; files[partition] = fopen(path.c_str(), "ab"); if (files[partition] == NULL) { Debug(Debug::ERROR) << "Cannot open bucket " << path << ": " << strerror(errno) << "\n"; EXIT(EXIT_FAILURE); } + openFiles.fetch_add(1); } if (fwrite(buffer.data(), sizeof(KmerRecord), buffer.size(), files[partition]) != buffer.size()) { // Name the reason: a full scratch filesystem is by far the likeliest way @@ -219,17 +276,13 @@ void KmerBucketWriter::flush(unsigned int partition) { << partition << " of " << dir << ": " << strerror(errno) << "\n"; EXIT(EXIT_FAILURE); } - // Checked, and before the buffer is dropped: buffered I/O can report ENOSPC, - // a quota, or a remote filesystem error only at close, and the queue marks the - // item done straight after flushAll(). An unchecked close here let an item be - // recorded complete with k-mers missing, which the reduce then reads as "this - // k-mer had no partner" -- a wrong clustering with no diagnostic. - if (fclose(files[partition]) != 0) { - Debug(Debug::ERROR) << "Cannot close k-mer bucket " << partition << " of " << dir << ": " - << strerror(errno) << "\n"; - EXIT(EXIT_FAILURE); + // Held open while the writer's descriptor budget allows, so a partition that + // is flushed once per work item does not pay an open and a close each time + // (see deriveDescriptorBudget). Past the budget this is the old + // open-write-close, which is what makes a 65536-partition run safe. + if (openFiles.load() > descriptorBudget) { + closeFile(partition); } - files[partition] = NULL; buffer.clear(); } @@ -258,6 +311,9 @@ uint64_t KmerBucketWriter::getRecordCount() { } void KmerBucketWriter::flushAll() { + // Descriptors that survive a flush now hold the records in a stdio buffer, so + // getting them to the OS -- which is exactly what this call owes its caller + // before the work item is marked done -- takes a checked fflush. for (unsigned int p = 0; p < partitionCount; p++) { std::lock_guard guard(mutexes[p]); flush(p); @@ -273,13 +329,7 @@ void KmerBucketWriter::close() { for (unsigned int p = 0; p < partitionCount; p++) { std::lock_guard guard(mutexes[p]); flush(p); - if (files[p] != NULL) { - if (fclose(files[p]) != 0) { - Debug(Debug::ERROR) << "Cannot close k-mer bucket for partition " << p << "\n"; - EXIT(EXIT_FAILURE); - } - files[p] = NULL; - } + closeFile(p); } } @@ -307,7 +357,22 @@ std::vector KmerBucketReader::shardFiles(const std::string &dir, un shards.push_back(path + "/" + name); } } - closedir(handle); + // errno was set to 0 before the loop precisely so it could be read here, and + // then never was. A directory read that fails part-way returns NULL exactly as + // the end of the directory does, so an unchecked loop turns an I/O error into a + // *partial* shard list -- which downstream reads as "these k-mers had no + // partner", a smaller clustering with no diagnostic. EdgeBucketWriter::shardFiles + // already does this; the two are now the same shape. + const int readErr = errno; + if (readErr != 0) { + Debug(Debug::ERROR) << "Cannot read " << path << ": " << strerror(readErr) << "\n"; + EXIT(EXIT_FAILURE); + } + if (closedir(handle) != 0) { + const int closeErr = errno; + Debug(Debug::ERROR) << "Cannot close " << path << ": " << strerror(closeErr) << "\n"; + EXIT(EXIT_FAILURE); + } // Sorted so a partition reads back in the same order on every run, which // keeps the whole pipeline reproducible regardless of directory order. std::sort(shards.begin(), shards.end()); @@ -339,13 +404,29 @@ uint64_t KmerBucketReader::countRecords(const std::string &dir, unsigned int par void KmerBucketReader::readPartition(const std::string &dir, unsigned int partition, std::vector &out) { const std::vector shards = shardFiles(dir, partition); + // Sized in one go rather than grown per shard: resize() on a vector holding + // hundreds of gigabytes reallocates and copies the whole array once per shard, + // and a partition has one shard per worker. + uint64_t total = 0; + std::vector counts(shards.size(), 0); + for (size_t i = 0; i < shards.size(); i++) { + counts[i] = FileUtil::getFileSize(shards[i]) / sizeof(KmerRecord); + total += counts[i]; + } + out.reserve(out.size() + static_cast(total)); for (size_t i = 0; i < shards.size(); i++) { + // Rounded down to whole records rather than refused, matching countRecords + // and readPartitionAsPositions. A torn tail is what an interrupted worker + // leaves; its item is redone into a *different* shard, so nothing is lost, + // and making it fatal meant every later read of that partition died on it + // forever with no recovery but deleting the file by hand. const size_t bytes = FileUtil::getFileSize(shards[i]); if (bytes % sizeof(KmerRecord) != 0) { - Debug(Debug::ERROR) << "Bucket " << shards[i] << " is truncated\n"; - EXIT(EXIT_FAILURE); + Debug(Debug::WARNING) << "Bucket " << shards[i] << " ends mid-record at " << bytes + << " bytes, as an interrupted worker leaves it; reading the " + << counts[i] << " whole records it holds.\n"; } - const size_t count = bytes / sizeof(KmerRecord); + const size_t count = counts[i]; if (count == 0) { continue; } diff --git a/src/linclust/KmerPartition.h b/src/linclust/KmerPartition.h index 59f057e42..410537ad6 100644 --- a/src/linclust/KmerPartition.h +++ b/src/linclust/KmerPartition.h @@ -1,6 +1,7 @@ #ifndef MMSEQS_KMERPARTITION_H #define MMSEQS_KMERPARTITION_H +#include #include #include #include @@ -236,6 +237,7 @@ class KmerBucketWriter { // Caller must hold mutexes[partition]. void flush(unsigned int partition); + void closeFile(unsigned int partition); std::string dir; std::string shardId; @@ -247,6 +249,11 @@ class KmerBucketWriter { std::vector recordCounts; unsigned int partitionFrom; unsigned int partitionTo; // exclusive + // Descriptors are kept open across flushes up to this many; past it a flush + // reverts to open-append-close. Counted atomically because the count is shared + // across the per-partition mutexes. + size_t descriptorBudget; + std::atomic openFiles; }; // Reads every shard of one partition back. diff --git a/src/linclust/PartitionSequences.cpp b/src/linclust/PartitionSequences.cpp index 3c79c18f8..087d6665c 100644 --- a/src/linclust/PartitionSequences.cpp +++ b/src/linclust/PartitionSequences.cpp @@ -23,7 +23,7 @@ const size_t DATA_COALESCE_BYTES = 1024 * 1024; } // namespace PartitionSequences::PartitionSequences(const std::string &dbName) - : dbName(dbName), dataFd(-1), indexFd(-1), bytesRead(0) { + : dbName(dbName), dataFd(-1), indexFd(-1), bytesRead(0), slotShift(0) { const DenseIndex::Info info = DenseIndex::readInfo(dbName); entryCount = info.entryCount; firstKey = info.firstKey; @@ -79,6 +79,7 @@ void PartitionSequences::load(const std::vector &sortedKeys) { lengths.assign(keys.size(), 0); arena.clear(); bytesRead = 0; + buildDirectory(); if (keys.empty()) { return; } @@ -137,13 +138,49 @@ void PartitionSequences::load(const std::vector &sortedKeys) { } } +void PartitionSequences::buildDirectory() { + slotStart.clear(); + slotShift = 0; + if (keys.empty()) { + return; + } + const uint64_t span = keys.back() - keys.front() + 1; + const uint64_t wantSlots = std::max(keys.size() / KEYS_PER_SLOT, 1); + while ((span >> slotShift) > wantSlots) { + slotShift++; + } + const size_t slots = static_cast(span >> slotShift) + 1; + // Filled from the back, so each slot ends up holding the *first* index at or + // past it and the array is non-decreasing without a second pass. + slotStart.assign(slots + 1, keys.size()); + for (size_t i = keys.size(); i > 0; i--) { + const size_t slot = static_cast((keys[i - 1] - keys.front()) >> slotShift); + slotStart[slot] = i - 1; + } + for (size_t s = slots; s > 0; s--) { + if (slotStart[s - 1] > slotStart[s]) { + slotStart[s - 1] = slotStart[s]; + } + } +} + const char *PartitionSequences::get(uint64_t key, unsigned int *length) const { - const std::vector::const_iterator it = - std::lower_bound(keys.begin(), keys.end(), key); - if (it == keys.end() || *it != key) { + if (keys.empty() || key < keys.front() || key > keys.back()) { return NULL; } - const size_t idx = static_cast(it - keys.begin()); - *length = lengths[idx]; - return arena.data() + offsets[idx]; + const size_t slot = static_cast((key - keys.front()) >> slotShift); + const size_t from = static_cast(slotStart[slot]); + const size_t to = static_cast(slotStart[slot + 1]); + // A handful of contiguous entries on average, so a linear scan beats a branchy + // search and stays in one or two cache lines. + for (size_t i = from; i < to; i++) { + if (keys[i] == key) { + *length = lengths[i]; + return arena.data() + offsets[i]; + } + if (keys[i] > key) { + break; + } + } + return NULL; } diff --git a/src/linclust/PartitionSequences.h b/src/linclust/PartitionSequences.h index 8faa0d8e4..2a0585173 100644 --- a/src/linclust/PartitionSequences.h +++ b/src/linclust/PartitionSequences.h @@ -45,6 +45,7 @@ class PartitionSequences { PartitionSequences &operator=(const PartitionSequences &); void readAt(int fd, void *dst, size_t length, size_t offset, const char *what); + void buildDirectory(); std::string dbName; int dataFd; @@ -57,6 +58,26 @@ class PartitionSequences { std::vector lengths; // residues, without the newline/terminator std::vector arena; uint64_t bytesRead; + + // Coarse key -> index directory, so get() is not a binary search. + // + // get() is called twice per candidate edge, and at 1e12 a bucket holds + // billions of keys: std::lower_bound over that is ~31 cache-missing probes + // each, which is a large fraction of the align stage's wall clock and + // touches nothing else in cache. + // + // Keys are dense, so `key - keys.front()` is very nearly the index already. + // Slotting on the high bits of that difference -- sized for about + // KEYS_PER_SLOT keys per slot -- turns the lookup into one directory read + // plus a short scan over contiguous keys. Indexing the difference *directly* + // is what the review suggested, but a bucket whose members reach far from its + // representatives would then allocate a slot per key of the whole span, which + // is unbounded; a slot per KEYS_PER_SLOT keys is one byte per loaded key. + static const uint64_t KEYS_PER_SLOT = 8; + uint64_t slotShift; + // slotStart[s] is the first index in `keys` whose slot is >= s, so a lookup + // scans [slotStart[s], slotStart[s + 1]). + std::vector slotStart; }; #endif diff --git a/src/linclust/alignparallel.cpp b/src/linclust/alignparallel.cpp index 8e52a799c..55db6c842 100644 --- a/src/linclust/alignparallel.cpp +++ b/src/linclust/alignparallel.cpp @@ -45,11 +45,15 @@ #include "Util.h" #include +#include #include #include #include #include +#include +#include + #ifdef OPENMP #include #endif @@ -131,20 +135,44 @@ size_t mergePairCopies(std::vector &edges) { // Waves are concatenated in order because each has its own queue over its own // contiguous slice of partition space, so wave w's items are partitions // [w * P / W, (w + 1) * P / W) and appending lands each at its own index. -std::vector readReduceAuthority(const std::string &edgeDir) { +std::vector readReduceAuthority(const std::string &edgeDir, unsigned int expectedWaves, + unsigned int expectedPartitions) { std::vector authority; - for (unsigned int wave = 0;; wave++) { + // Every wave, by count, not "until one is missing". + // + // This used to stop at the first absent reduce..queue and accept whatever + // it had. Starting the align before the last wave had run therefore produced a + // short authority vector that looked complete: every edge block naming a + // partition past its end was kept (see EdgeBucketReader::readShard), the + // missing partitions' edges simply were not there, and the result was a + // successful but incomplete clustering that a restart would then preserve, + // because the align output now looked finished too. + for (unsigned int wave = 0; wave < expectedWaves; wave++) { std::vector workers; const std::string path = edgeDir + "/coord/reduce." + SSTR(wave) + ".queue"; if (WorkQueue::readCompletedWorkers(path, workers) == false) { - break; + Debug(Debug::ERROR) << "The reduce recorded " << expectedWaves << " waves in " + << edgeDir << "/coord/edge.info, but " << path + << " does not exist. Run kmerreduceparallel for every wave " + << "before aligning.\n"; + EXIT(EXIT_FAILURE); + } + for (size_t i = 0; i < workers.size(); i++) { + if (workers[i] < 0) { + Debug(Debug::ERROR) << "Partition " << (authority.size() + i) << " of " << path + << " was never completed. The reduce is unfinished; re-run it " + << "before aligning.\n"; + EXIT(EXIT_FAILURE); + } } authority.insert(authority.end(), workers.begin(), workers.end()); } - if (authority.empty()) { - Debug(Debug::WARNING) << "No reduce work queue under " << edgeDir - << "/coord; cannot tell a crashed worker's superseded edges from a " - << "second partition's. Duplicate support would be summed.\n"; + if (authority.size() != expectedPartitions) { + Debug(Debug::ERROR) << "The reduce queues under " << edgeDir << "/coord cover " + << authority.size() << " partitions, but edge.info records " + << expectedPartitions << ". The edge directory mixes output from two " + << "different runs.\n"; + EXIT(EXIT_FAILURE); } return authority; } @@ -167,65 +195,185 @@ size_t readBucket(const std::string &edgeDir, unsigned int bucket, // 902,795. // // t is a pass-1 representative, so it is dense in *sub-key* space -- which makes the -// lookup a CSR index over sub-keys rather than anything indexed by the full key -// space: an offset per representative plus one key per sequence. +// lookup a CSR index over sub-keys: an offset per representative plus one key per +// sequence. +// +// That CSR is built once by createrepdb and *paged*, not loaded. Every align +// worker used to build it for itself -- the whole key map (8 B x R), an offset per +// representative (8 B x R) and every pass-1 member key (8 B x N) -- by streaming +// the pass-1 TSV twice with a binary search per line. At 1e11 with R/N = 0.4 that +// is ~1.44 TB resident per worker on a 2 TB node, before the bucket's own edges +// and sequence arena, and ~14.4 TB at 1e12; the two TSV passes were themselves +// 2 x 1e11 cache-missing probes into a 320 GB array, in every worker, over a +// ~2.8 TB file. +// +// Now a worker reads only the slice its current bucket refers to. Resident cost is +// O(bucket), and the whole thing is one sequential pread per contiguous run of +// sub-keys because the bucket's member sub-keys are already sorted. struct FilterGate { - std::vector keymap; // sub-key -> original key - std::vector clusterStart; // sub-key -> offset into members - std::vector members; // original keys, grouped by pass-1 cluster + // The bucket's slice, rebuilt per bucket by loadSlice(). + std::vector subs; // sorted, distinct member sub-keys of this bucket + std::vector selfKey; // keymap[sub], parallel to subs + std::vector start; // index into `members`, parallel to subs + std::vector count; // cluster size, parallel to subs + std::vector members; // the original keys of those clusters, concatenated PartitionSequences fullSeqs; - FilterGate(const std::string &fullDb) : fullSeqs(fullDb) {} + uint64_t repCount; // sub-keys the gate covers + uint64_t memberCount; // rows in the members file + int keymapFd; + int offsetsFd; + int membersFd; + + FilterGate(const std::string &fullDb) + : fullSeqs(fullDb), repCount(0), memberCount(0), keymapFd(-1), offsetsFd(-1), + membersFd(-1) {} - size_t size(uint64_t sub) const { return clusterStart[sub + 1] - clusterStart[sub]; } + ~FilterGate() { + if (keymapFd >= 0) close(keymapFd); + if (offsetsFd >= 0) close(offsetsFd); + if (membersFd >= 0) close(membersFd); + } - void load(const std::string &keymapFile, const std::string &pass1Tsv) { - const size_t bytes = FileUtil::getFileSize(keymapFile); - keymap.resize(bytes / sizeof(uint64_t)); - FILE *m = FileUtil::openFileOrDie(keymapFile.c_str(), "rb", true); - if (fread(keymap.data(), sizeof(uint64_t), keymap.size(), m) != keymap.size()) { - Debug(Debug::ERROR) << "Cannot read " << keymapFile << "\n"; + static int openReadOnly(const std::string &path) { + const int fd = ::open(path.c_str(), O_RDONLY); + if (fd < 0) { + Debug(Debug::ERROR) << "Cannot open " << path << ": " << strerror(errno) << "\n"; EXIT(EXIT_FAILURE); } - fclose(m); - - // Two streaming passes: count, then fill. The representative of a pass-1 - // cluster is looked up in the ascending key map by binary search. - std::vector counts(keymap.size() + 1, 0); - for (int pass = 0; pass < 2; pass++) { - FILE *f = FileUtil::openFileOrDie(pass1Tsv.c_str(), "r", true); - char *line = NULL; - size_t cap = 0; - while (getline(&line, &cap, f) > 0) { - char *tab = strchr(line, '\t'); - if (tab == NULL) continue; - const uint64_t rep = strtoull(line, NULL, 10); - const uint64_t member = strtoull(tab + 1, NULL, 10); - const std::vector::const_iterator it = - std::lower_bound(keymap.begin(), keymap.end(), rep); - if (it == keymap.end() || *it != rep) continue; - const size_t sub = static_cast(it - keymap.begin()); - if (pass == 0) { - counts[sub]++; - } else { - members[clusterStart[sub] + counts[sub]] = member; - counts[sub]++; - } + return fd; + } + + static void readAt(int fd, void *dst, size_t bytes, size_t offset, const std::string &what) { + char *p = static_cast(dst); + size_t done = 0; + while (done < bytes) { + const ssize_t got = pread(fd, p + done, bytes - done, static_cast(offset + done)); + if (got <= 0) { + if (got < 0 && errno == EINTR) continue; + Debug(Debug::ERROR) << "Cannot read " << what << ": " + << (got < 0 ? strerror(errno) : "unexpected end of file") + << "\n"; + EXIT(EXIT_FAILURE); } - free(line); - fclose(f); - if (pass == 0) { - clusterStart.assign(keymap.size() + 1, 0); - uint64_t total = 0; - for (size_t i = 0; i < keymap.size(); i++) { - clusterStart[i] = total; - total += counts[i]; - } - clusterStart[keymap.size()] = total; - members.assign(total, 0); - counts.assign(keymap.size() + 1, 0); + done += static_cast(got); + } + } + + // Opens the CSR and validates it against the key map, without reading either. + void open(const std::string &keymapFile, const std::string &repDb) { + const std::string offsetsPath = repDb + ".gate.offsets"; + const std::string membersPath = repDb + ".gate.members"; + if (FileUtil::fileExists(offsetsPath.c_str()) == false) { + Debug(Debug::ERROR) << "No filter gate next to " << repDb << " (" << offsetsPath + << " is missing). It is built by createrepdb; re-run that stage " + << "with this build.\n"; + EXIT(EXIT_FAILURE); + } + const size_t keymapBytes = FileUtil::getFileSize(keymapFile); + repCount = keymapBytes / sizeof(uint64_t); + const size_t offsetBytes = FileUtil::getFileSize(offsetsPath); + if (offsetBytes != (repCount + 1) * sizeof(uint64_t)) { + Debug(Debug::ERROR) << offsetsPath << " holds " << (offsetBytes / sizeof(uint64_t)) + << " offsets but " << keymapFile << " holds " << repCount + << " representatives. They are from different runs of " + << "createrepdb.\n"; + EXIT(EXIT_FAILURE); + } + memberCount = FileUtil::getFileSize(membersPath) / sizeof(uint64_t); + keymapFd = openReadOnly(keymapFile); + offsetsFd = openReadOnly(offsetsPath); + membersFd = openReadOnly(membersPath); + } + + // Fetches the clusters of `wanted` (sorted, distinct sub-keys) and reports the + // original keys they contain, so the caller can prefetch those sequences. + void loadSlice(const std::vector &wanted, std::vector &memberKeysOut) { + subs = wanted; + selfKey.assign(subs.size(), 0); + start.assign(subs.size(), 0); + count.assign(subs.size(), 0); + members.clear(); + memberKeysOut.clear(); + if (subs.empty()) { + return; + } + + // Offsets and key map, in coalesced ascending runs. Both are fixed width + // and indexed by sub-key, so a run of nearby sub-keys is one pread. + const size_t coalesce = 64 * 1024 / sizeof(uint64_t); + std::vector block; + size_t i = 0; + while (i < subs.size()) { + size_t j = i; + while (j + 1 < subs.size() && subs[j + 1] - subs[i] < coalesce) { + j++; } + // One past the last, because a cluster needs offsets[s] and offsets[s+1]. + const uint64_t from = subs[i]; + const uint64_t to = subs[j] + 2; + block.resize(static_cast(to - from)); + readAt(offsetsFd, block.data(), block.size() * sizeof(uint64_t), + static_cast(from) * sizeof(uint64_t), "the filter gate offsets"); + for (size_t k = i; k <= j; k++) { + const size_t at = static_cast(subs[k] - from); + start[k] = block[at]; + count[k] = block[at + 1] - block[at]; + } + block.resize(static_cast(subs[j] - subs[i] + 1)); + readAt(keymapFd, block.data(), block.size() * sizeof(uint64_t), + static_cast(subs[i]) * sizeof(uint64_t), "the sub-key map"); + for (size_t k = i; k <= j; k++) { + selfKey[k] = block[static_cast(subs[k] - subs[i])]; + } + i = j + 1; + } + + // The member keys themselves. Clusters of ascending sub-keys occupy + // ascending, mostly contiguous ranges of the members file, so this is a + // forward scan; runs that are actually adjacent become one read. + uint64_t total = 0; + for (size_t k = 0; k < subs.size(); k++) { + total += count[k]; + } + members.resize(static_cast(total)); + uint64_t at = 0; + i = 0; + while (i < subs.size()) { + size_t j = i; + while (j + 1 < subs.size() && start[j + 1] == start[j] + count[j]) { + j++; + } + uint64_t run = 0; + for (size_t k = i; k <= j; k++) { + run += count[k]; + } + if (run > 0) { + readAt(membersFd, members.data() + at, static_cast(run) * sizeof(uint64_t), + static_cast(start[i]) * sizeof(uint64_t), "the filter gate members"); + } + // start[] is rewritten as an index into the local arena. + uint64_t local = at; + for (size_t k = i; k <= j; k++) { + start[k] = local; + local += count[k]; + } + at += run; + i = j + 1; + } + + memberKeysOut.assign(members.begin(), members.end()); + } + + // Index into the loaded slice for a sub-key, or -1 when the bucket did not ask + // for it. + int64_t find(uint64_t sub) const { + const std::vector::const_iterator it = + std::lower_bound(subs.begin(), subs.end(), sub); + if (it == subs.end() || *it != sub) { + return -1; } + return static_cast(it - subs.begin()); } }; @@ -245,14 +393,23 @@ struct FilterGate { bool passesFilterGate(const FilterGate *gate, uint64_t subMember, Sequence &query, Sequence &element, BlockAligner &aligner, Matcher &matcher, Parameters &par, unsigned int swMode, short elementDiagonal) { - if (gate == NULL || subMember >= gate->keymap.size()) { + if (gate == NULL || subMember >= gate->repCount) { + return true; + } + const int64_t slot = gate->find(subMember); + if (slot < 0) { + // Not in this bucket's slice. The slice is built from exactly the member + // sub-keys of the bucket's edges, so this is only reachable for a sub-key + // no edge named -- for which the gate is never consulted. return true; } - if (gate->size(subMember) <= 1) { + if (gate->count[static_cast(slot)] <= 1) { return true; // stock only runs the loop when numClu > 1 } - const uint64_t targetKey = gate->keymap[subMember]; - for (uint64_t j = gate->clusterStart[subMember]; j < gate->clusterStart[subMember + 1]; j++) { + const uint64_t targetKey = gate->selfKey[static_cast(slot)]; + const uint64_t from = gate->start[static_cast(slot)]; + const uint64_t to = from + gate->count[static_cast(slot)]; + for (uint64_t j = from; j < to; j++) { const uint64_t elementKey = gate->members[j]; if (elementKey == targetKey) { continue; @@ -326,14 +483,19 @@ int alignparallel(int argc, const char **argv, const Command &command) { EXIT(EXIT_FAILURE); } unsigned int bucketCount = 0; + uint64_t bucketSpan = 0; + unsigned int partitionCount = 0; + unsigned int waveCount = 0; { FILE *file = FileUtil::openFileOrDie(manifestPath.c_str(), "r", true); char name[64]; size_t value; while (fscanf(file, "%63s\t%zu\n", name, &value) == 2) { - if (std::string(name) == "bucketCount") { - bucketCount = static_cast(value); - } + const std::string key = name; + if (key == "bucketCount") bucketCount = static_cast(value); + else if (key == "bucketSpan") bucketSpan = value; + else if (key == "partitionCount") partitionCount = static_cast(value); + else if (key == "waveCount") waveCount = static_cast(value); } fclose(file); } @@ -341,6 +503,13 @@ int alignparallel(int argc, const char **argv, const Command &command) { Debug(Debug::ERROR) << "Edge manifest " << manifestPath << " has no bucket count\n"; EXIT(EXIT_FAILURE); } + if (partitionCount == 0 || waveCount == 0) { + Debug(Debug::ERROR) << "Edge manifest " << manifestPath << " does not record how many " + << "partitions and waves the reduce covered, so this stage cannot " + << "tell a finished reduce from a partial one. It was written by an " + << "older build; re-run kmerreduceparallel with this one.\n"; + EXIT(EXIT_FAILURE); + } par.printParameters(command.cmd, argc, argv, *command.params); if (FileUtil::directoryExists(alnDir.c_str()) == false) { @@ -379,9 +548,14 @@ int alignparallel(int argc, const char **argv, const Command &command) { EXIT(EXIT_FAILURE); } gate = new FilterGate(par.filterSeqDBFile); - gate->load(par.keyMapFile, par.filterCluDBFile); - Debug(Debug::INFO) << "Filter gate: " << gate->keymap.size() << " representatives, " - << gate->members.size() << " pass-1 members\n"; + // seqDb is the re-keyed representative database this pass runs on, which + // is where createrepdb put the CSR. --filter-cludb-file is no longer read + // here at all: it is the input the CSR was built from, kept as a parameter + // so the stage still states which clustering it is gating against. + gate->open(par.keyMapFile, seqDb); + Debug(Debug::INFO) << "Filter gate: " << gate->repCount << " representatives, " + << gate->memberCount << " pass-1 members, paged from " + << seqDb << ".gate.*\n"; } Debug(Debug::INFO) << "Aligning " << bucketCount << " edge buckets; score-per-column cutoff " << scorePerColThreshold << "\n"; @@ -392,7 +566,33 @@ int alignparallel(int argc, const char **argv, const Command &command) { PartitionSequences sequences(seqDb); uint64_t survivorCount = 0; - const std::vector reduceAuthority = readReduceAuthority(edgeDir); + const std::vector reduceAuthority = + readReduceAuthority(edgeDir, waveCount, partitionCount); + + // The layout the greedy sweep must consume, recorded next to the output rather + // than rediscovered from it. greedycluster used to infer the bucket count by + // counting consecutive p.edges files, so an align that had only produced a + // prefix read as a complete, smaller layout -- and it recomputed bucketSpan + // from that smaller count, silently sweeping the wrong key ranges. + { + FileLock layoutLock(coordDir + "/align.lock"); + layoutLock.lock(); + const std::string layoutPath = coordDir + "/align.info"; + if (FileUtil::fileExists(layoutPath.c_str()) == false) { + const std::string tmp = layoutPath + ".tmp." + SSTR(getpid()); + FILE *f = FileUtil::openAndDelete(tmp.c_str(), "w"); + fprintf(f, "bucketCount\t%zu\n", (size_t)bucketCount); + fprintf(f, "bucketSpan\t%zu\n", (size_t)bucketSpan); + fprintf(f, "entryCount\t%zu\n", (size_t)info.entryCount); + if (fclose(f) != 0 || rename(tmp.c_str(), layoutPath.c_str()) != 0) { + Debug(Debug::ERROR) << "Cannot publish " << layoutPath << ": " << strerror(errno) + << "\n"; + layoutLock.unlock(); + EXIT(EXIT_FAILURE); + } + } + layoutLock.unlock(); + } { WorkQueue queue(coordDir + "/align.queue", static_cast(bucketCount)); @@ -425,14 +625,24 @@ int alignparallel(int argc, const char **argv, const Command &command) { // The gate compares against the pass-1 cluster members of each target, // which live in the *full* database; fetch exactly those, in key order. if (gate != NULL) { - std::vector gateKeys; + // The bucket's member sub-keys, deduplicated *before* their + // clusters are expanded. Expanding first and deduplicating after + // -- which is what this used to do -- materialised one key per + // (edge, cluster member) pair, so a bucket referring to a few + // large pass-1 clusters many times over held far more than the + // clusters themselves. + std::vector wanted; + wanted.reserve(edges.size()); for (size_t i = 0; i < edges.size(); i++) { const uint64_t sub = edges[i].getMember(); - if (sub >= gate->keymap.size()) continue; - for (uint64_t j = gate->clusterStart[sub]; j < gate->clusterStart[sub + 1]; j++) { - gateKeys.push_back(gate->members[j]); + if (sub < gate->repCount) { + wanted.push_back(sub); } } + SORT_PARALLEL(wanted.begin(), wanted.end()); + wanted.erase(std::unique(wanted.begin(), wanted.end()), wanted.end()); + std::vector gateKeys; + gate->loadSlice(wanted, gateKeys); SORT_PARALLEL(gateKeys.begin(), gateKeys.end()); gateKeys.erase(std::unique(gateKeys.begin(), gateKeys.end()), gateKeys.end()); gate->fullSeqs.load(gateKeys); diff --git a/src/linclust/createrepdb.cpp b/src/linclust/createrepdb.cpp index ad9050cb1..f4a1721dc 100644 --- a/src/linclust/createrepdb.cpp +++ b/src/linclust/createrepdb.cpp @@ -20,10 +20,35 @@ * descending length order; the i-th representative in ascending original-key order * is exactly sub-key i. No sort is needed, and the new database is a sequential * copy rather than a gather. + * + * Memory is O(N/8 + R/CHUNK_ENTRIES), not O(R) + * ------------------------------------------- + * The copy used to materialise, for every representative, a packed source index + * entry (12 B), a destination offset (8 B), a length (4 B) and a key-map slot + * (8 B). At the 37% representative rate measured on the 5B run that is ~1.29 TB at + * 1e11 and ~12.9 TB at 1e12, on a node with 2 TB -- so the stage could not run at + * the scale the pipeline exists for. + * + * None of it has to be resident. Destination offsets are a prefix sum, and a + * prefix sum only needs a checkpoint every CHUNK_ENTRIES representatives: the + * chunk that owns a representative recomputes the offsets inside itself from its + * own checkpoint. Source entries and lengths come back out of the source + * `.index.bin`, which the chunk reads sequentially anyway. The key map is written + * straight to its file as the plan pass discovers it. + * + * What is left is two 8-byte checkpoints per chunk -- 195 MB at 1e12 with + * CHUNK_ENTRIES = 8192 -- plus the representative bitmap at N/8 (125 GB at 1e12, + * which is what the whole pipeline already budgets for a key-space bitmap). + * + * The cost is one extra sequential pass over the source index, since the copy + * re-reads what the plan pass already saw. That is 12 B per source entry against + * the 32 B per representative it replaces, and it is streaming rather than + * resident. */ #include "Command.h" #include "Debug.h" #include "DenseIndex.h" +#include "FastSort.h" #include "FileUtil.h" #include "Parameters.h" #include "Util.h" @@ -43,15 +68,69 @@ namespace { +// Representatives per copy chunk. Sets both the checkpoint density (two 8-byte +// values per chunk) and the size of a worker's buffers (one index entry and one +// sequence per representative), so it trades resident memory against I/O size. +// 8192 gives ~2 MB of sequence data per chunk on protein input, which is already +// a comfortable write, and 195 MB of checkpoints at 1e12. +const uint64_t CHUNK_ENTRIES = 8192; + class Bitmap { public: explicit Bitmap(uint64_t bits) : words((bits + 63) / 64, 0) {} bool get(uint64_t i) const { return (words[i >> 6] >> (i & 63)) & 1ULL; } void set(uint64_t i) { words[i >> 6] |= 1ULL << (i & 63); } + // Sets the bit and reports whether it was this call that set it. Used by the + // threaded TSV scan below, where several threads can meet the same + // representative and exactly one of them must count it. + bool testAndSet(uint64_t i) { + const uint64_t mask = 1ULL << (i & 63); + const uint64_t previous = __sync_fetch_and_or(&words[i >> 6], mask); + return (previous & mask) == 0; + } uint64_t bytes() const { return words.size() * sizeof(uint64_t); } + // Number of set bits below `i`. This is exactly "the sub-key of original key + // i", because sub-keys are assigned to representatives in ascending original + // key order. + // + // Backed by one checkpoint per RANK_BLOCK keys, so it costs 8 bytes per 4096 + // keys -- 2 GB at 1e12 against the 125 GB the bitmap itself already needs -- + // and a lookup is one array read plus at most 64 popcounts. Holding a + // key -> sub-key table instead would be 8 bytes per *key*. + static const uint64_t RANK_BLOCK = 4096; + + void buildRank() { + rank.assign(words.size() / (RANK_BLOCK / 64) + 1, 0); + uint64_t total = 0; + for (size_t w = 0; w < words.size(); w++) { + if (w % (RANK_BLOCK / 64) == 0) { + rank[w / (RANK_BLOCK / 64)] = total; + } + total += static_cast(__builtin_popcountll(words[w])); + } + } + + uint64_t rankOf(uint64_t i) const { + const size_t word = static_cast(i >> 6); + const size_t block = word / (RANK_BLOCK / 64); + uint64_t count = rank[block]; + for (size_t w = block * (RANK_BLOCK / 64); w < word; w++) { + count += static_cast(__builtin_popcountll(words[w])); + } + const uint64_t bit = i & 63; + if (bit > 0) { + count += static_cast( + __builtin_popcountll(words[word] & ((1ULL << bit) - 1))); + } + return count; + } + + uint64_t rankBytes() const { return rank.size() * sizeof(uint64_t); } + private: std::vector words; + std::vector rank; }; int openOrDie(const std::string &path, int flags) { @@ -100,6 +179,80 @@ void allocate(const std::string &path, uint64_t size) { close(fd); } +// Where each chunk of CHUNK_ENTRIES representatives begins, on both sides. +// +// srcRow[c] is the source row holding the chunk's first representative, so the +// chunk can seek straight there instead of scanning from zero. dataOffset[c] is +// the destination byte that representative starts at, so the chunk recomputes the +// offsets inside itself and nothing global has to be held. +struct CopyPlan { + std::vector srcRow; + std::vector dataOffset; + uint64_t totalBytes; + uint32_t maxLen; + + CopyPlan() : totalBytes(0), maxLen(0) {} +}; + +// One streaming pass over the source index. Optionally emits the sub-key -> +// original-key map as it goes, which is what keeps that 8 B per representative +// off the heap. +CopyPlan planCopy(int srcIdx, const Bitmap &keep, uint64_t entryCount, uint64_t keptCount, + FILE *keyMapOut, const std::string &keyMapPath) { + CopyPlan plan; + plan.srcRow.reserve(static_cast(keptCount / CHUNK_ENTRIES + 2)); + plan.dataOffset.reserve(static_cast(keptCount / CHUNK_ENTRIES + 2)); + + const uint64_t block = 1 << 20; + std::vector buf(block); + std::vector keyBuf; + keyBuf.reserve(1 << 16); + + uint64_t kept = 0; + uint64_t total = 0; + for (uint64_t from = 0; from < entryCount; from += block) { + const uint64_t n = std::min(block, entryCount - from); + readAt(srcIdx, buf.data(), n * sizeof(DenseIndex::Entry), DenseIndex::entryOffset(from), + "source index"); + for (uint64_t i = 0; i < n; i++) { + if (keep.get(from + i) == false) continue; + if (kept % CHUNK_ENTRIES == 0) { + plan.srcRow.push_back(from + i); + plan.dataOffset.push_back(total); + } + total += buf[i].length; + plan.maxLen = std::max(plan.maxLen, buf[i].length); + if (keyMapOut != NULL) { + keyBuf.push_back(from + i); + if (keyBuf.size() == keyBuf.capacity()) { + if (fwrite(keyBuf.data(), sizeof(uint64_t), keyBuf.size(), keyMapOut) + != keyBuf.size()) { + Debug(Debug::ERROR) << "Cannot write " << keyMapPath << ": " + << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + keyBuf.clear(); + } + } + kept++; + } + } + if (keyMapOut != NULL && keyBuf.empty() == false) { + if (fwrite(keyBuf.data(), sizeof(uint64_t), keyBuf.size(), keyMapOut) != keyBuf.size()) { + Debug(Debug::ERROR) << "Cannot write " << keyMapPath << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + } + if (kept != keptCount) { + Debug(Debug::ERROR) << "Counted " << keptCount << " representatives in the clustering but " + << kept << " in the index. The clustering and the database do not " + << "match.\n"; + EXIT(EXIT_FAILURE); + } + plan.totalBytes = total; + return plan; +} + // Copies the entries flagged in `keep` into a new dense database, preserving key // order. Returns the total data bytes and the longest entry. struct CopyResult { @@ -109,67 +262,77 @@ struct CopyResult { CopyResult copyFlagged(const std::string &srcDb, const std::string &dstDb, const Bitmap &keep, uint64_t entryCount, uint64_t keptCount, int threads, - std::vector *keyMap) { + FILE *keyMapOut, const std::string &keyMapPath) { const int srcIdx = openOrDie(DenseIndex::fileName(srcDb), O_RDONLY); const int srcData = openOrDie(srcDb, O_RDONLY); - // Pass 1: per-key kept lengths, so the destination offsets are a prefix sum. - // The index is read sequentially in blocks rather than randomly per key. - std::vector offsets(keptCount + 1, 0); - std::vector lengths(keptCount, 0); - std::vector srcEntries(keptCount); - { - const uint64_t block = 1 << 20; - std::vector buf(block); - uint64_t out = 0; - for (uint64_t from = 0; from < entryCount; from += block) { - const uint64_t n = std::min(block, entryCount - from); - readAt(srcIdx, buf.data(), n * sizeof(DenseIndex::Entry), - DenseIndex::entryOffset(from), "source index"); - for (uint64_t i = 0; i < n; i++) { - if (keep.get(from + i) == false) continue; - srcEntries[out] = buf[i]; - lengths[out] = buf[i].length; - if (keyMap != NULL) (*keyMap)[out] = from + i; - out++; - } - } - } + const CopyPlan plan = planCopy(srcIdx, keep, entryCount, keptCount, keyMapOut, keyMapPath); + CopyResult result; - result.maxLen = 0; - uint64_t total = 0; - for (uint64_t i = 0; i < keptCount; i++) { - offsets[i] = total; - total += lengths[i]; - result.maxLen = std::max(result.maxLen, lengths[i]); - } - offsets[keptCount] = total; - result.dataBytes = total; + result.maxLen = plan.maxLen; + result.dataBytes = plan.totalBytes; - allocate(dstDb, total); - DenseIndex::createEmpty(dstDb, keptCount, 0, total, result.maxLen); + allocate(dstDb, plan.totalBytes); + DenseIndex::createEmpty(dstDb, keptCount, 0, plan.totalBytes, plan.maxLen); const int dstData = openOrDie(dstDb, O_WRONLY); const int dstIdx = openOrDie(DenseIndex::fileName(dstDb), O_WRONLY); - // Pass 2: copy. Entries are independent and both sides are in ascending offset - // order, so this is a threaded forward scan rather than a gather. + const int64_t chunkCount = static_cast(plan.srcRow.size()); + + // Pass 2: copy, one chunk of CHUNK_ENTRIES representatives at a time. Chunks + // are independent -- their destination ranges are disjoint and their source + // ranges are ascending -- so this is a threaded forward scan rather than a + // gather, exactly as before; what changed is that a chunk re-reads its own + // slice of the source index instead of every chunk's slice being held. #pragma omp parallel num_threads(threads) { std::vector buf; std::vector span; + std::vector entries; std::vector idxBuf; - const uint64_t chunk = 4096; + std::vector scanBuf(4096); #pragma omp for schedule(dynamic, 1) - for (uint64_t start = 0; start < keptCount; start += chunk) { - const uint64_t stop = std::min(start + chunk, keptCount); - const uint64_t bytes = offsets[stop] - offsets[start]; - buf.resize(static_cast(bytes)); - idxBuf.resize(static_cast(stop - start)); - for (uint64_t i = start; i < stop; i++) { - idxBuf[i - start].offset = offsets[i]; - idxBuf[i - start].length = lengths[i]; + for (int64_t c = 0; c < chunkCount; c++) { + const uint64_t keptFrom = static_cast(c) * CHUNK_ENTRIES; + const uint64_t count = std::min(CHUNK_ENTRIES, keptCount - keptFrom); + + // Collect this chunk's source entries, starting at the row the plan + // recorded. A run of representatives is contiguous in the *kept* + // numbering but sparse in the source, so this reads forward until it + // has `count` of them. + entries.clear(); + entries.reserve(static_cast(count)); + uint64_t row = plan.srcRow[static_cast(c)]; + while (entries.size() < count) { + const uint64_t n = std::min(scanBuf.size(), entryCount - row); + if (n == 0) { + Debug(Debug::ERROR) << "Ran out of source entries copying " << srcDb + << " chunk " << c << "\n"; + EXIT(EXIT_FAILURE); + } + readAt(srcIdx, scanBuf.data(), static_cast(n) * sizeof(DenseIndex::Entry), + DenseIndex::entryOffset(row), "source index"); + for (uint64_t i = 0; i < n && entries.size() < count; i++) { + if (keep.get(row + i)) { + entries.push_back(scanBuf[static_cast(i)]); + } + } + row += n; + } + + // Destination offsets, recomputed inside the chunk from its checkpoint. + const uint64_t base = plan.dataOffset[static_cast(c)]; + idxBuf.resize(static_cast(count)); + uint64_t at = base; + for (uint64_t k = 0; k < count; k++) { + idxBuf[static_cast(k)].offset = at; + idxBuf[static_cast(k)].length = entries[static_cast(k)].length; + at += entries[static_cast(k)].length; } + const uint64_t bytes = at - base; + buf.resize(static_cast(bytes)); + // Coalesced reads. Representatives are a *subset* of the source, so // one pread per entry meant 353M random reads at 1e9 and made this // stage 4x stock's createsubdb. Consecutive representatives are only a @@ -181,32 +344,34 @@ CopyResult copyFlagged(const std::string &srcDb, const std::string &dstDb, const // sequences too, and without a cap a database whose representatives // are thinly spread would read the whole file. That bounds the waste // at 2x while still collapsing dense regions into single reads. - uint64_t i = start; - while (i < stop) { - const uint64_t from = srcEntries[i].offset; - uint64_t to = from + lengths[i]; - uint64_t wanted = lengths[i]; + uint64_t i = 0; + while (i < count) { + const uint64_t from = entries[static_cast(i)].offset; + uint64_t to = from + entries[static_cast(i)].length; + uint64_t wanted = entries[static_cast(i)].length; uint64_t j = i + 1; - while (j < stop) { - const uint64_t end = srcEntries[j].offset + lengths[j]; - if (end - from > 2 * (wanted + lengths[j])) { + while (j < count) { + const uint64_t end = entries[static_cast(j)].offset + + entries[static_cast(j)].length; + if (end - from > 2 * (wanted + entries[static_cast(j)].length)) { break; } to = end; - wanted += lengths[j]; + wanted += entries[static_cast(j)].length; j++; } span.resize(static_cast(to - from)); readAt(srcData, span.data(), static_cast(to - from), from, "source data"); for (uint64_t k = i; k < j; k++) { - memcpy(buf.data() + (offsets[k] - offsets[start]), - span.data() + (srcEntries[k].offset - from), lengths[k]); + memcpy(buf.data() + (idxBuf[static_cast(k)].offset - base), + span.data() + (entries[static_cast(k)].offset - from), + entries[static_cast(k)].length); } i = j; } - writeAt(dstData, buf.data(), static_cast(bytes), offsets[start], "database"); + writeAt(dstData, buf.data(), static_cast(bytes), base, "database"); writeAt(dstIdx, idxBuf.data(), idxBuf.size() * sizeof(DenseIndex::Entry), - DenseIndex::entryOffset(start), "index"); + DenseIndex::entryOffset(keptFrom), "index"); } } @@ -217,6 +382,350 @@ CopyResult copyFlagged(const std::string &srcDb, const std::string &dstDb, const return result; } +// --------------------------------------------------------------------------- +// The pass-2 filter gate, as a CSR file rather than a per-worker vector. +// +// alignparallel's pass-2 gate needs, for a candidate member sub-key t, every +// original key in t's pass-1 cluster (Align2clust.cpp:657-730, the `allpass` +// loop). Every align worker used to build that itself: the whole sub-key -> +// original-key map (8 B x R), a cluster offset per representative (8 B x R) and +// every pass-1 member key (8 B x N), by streaming the pass-1 TSV twice and +// binary-searching the key map once per line. At 1e11 with R/N = 0.4 that is +// ~1.44 TB resident *per worker* on a 2 TB node -- before the bucket's own edges +// and sequence arena -- and ~14.4 TB at 1e12. It is the reason the stage could +// not run at either target. +// +// None of it is per-worker state: it is one function of clu1.tsv and the +// representative set, both of which this stage already has in hand. Built once +// here, in sub-key order, an align worker preads only the offsets and members of +// the sub-keys its own bucket touches -- O(bucket) instead of O(N). +// +// Layout, both fixed width so a slice is addressable by pread: +// .gate.offsets (R + 1) x uint64, offsets[s] .. offsets[s+1] is the +// member range of sub-key s +// .gate.members the original keys, grouped by sub-key, ascending +// within a group so the file is reproducible +// .gate.info the two counts, for validation on the reading side +// --------------------------------------------------------------------------- + +struct __attribute__((__packed__)) GateRow { + uint64_t sub; // sub-key of the pass-1 representative + uint64_t member; // original key of one of its members +}; + +// Thread-private spill shards, so the scan of clu1.tsv needs no locking. Same +// shape as translatekeys' pass-2 writers: .t.. +class GateSpill { +public: + GateSpill(const std::string &prefix, unsigned int buckets, size_t rowsPerBuffer) + : prefix(prefix), buckets(buckets), rowsPerBuffer(rowsPerBuffer), closed(false) { + opened.assign(buckets, false); + buffers.resize(buckets); + for (unsigned int b = 0; b < buckets; b++) { + buffers[b].reserve(rowsPerBuffer); + } + } + ~GateSpill() { close(); } + + void append(unsigned int b, uint64_t sub, uint64_t member) { + GateRow row; + row.sub = sub; + row.member = member; + buffers[b].push_back(row); + if (buffers[b].size() >= rowsPerBuffer) { + flush(b); + } + } + + void close() { + if (closed) return; + closed = true; + for (unsigned int b = 0; b < buckets; b++) { + flush(b); + } + } + + static std::string path(const std::string &prefix, unsigned int b) { + return prefix + "." + SSTR(b); + } + + static void read(const std::string &prefix, unsigned int b, std::vector &out) { + const std::string p = path(prefix, b); + if (FileUtil::fileExists(p.c_str()) == false) return; + const size_t bytes = FileUtil::getFileSize(p); + if (bytes == 0) return; + if (bytes % sizeof(GateRow) != 0) { + Debug(Debug::ERROR) << "Gate spill " << p << " is " << bytes << " bytes, not a whole " + << "number of rows. Remove it and re-run createrepdb.\n"; + EXIT(EXIT_FAILURE); + } + const size_t count = bytes / sizeof(GateRow); + const size_t at = out.size(); + out.resize(at + count); + FILE *f = FileUtil::openFileOrDie(p.c_str(), "rb", true); + if (fread(out.data() + at, sizeof(GateRow), count, f) != count) { + Debug(Debug::ERROR) << "Cannot read " << p << "\n"; + EXIT(EXIT_FAILURE); + } + fclose(f); + } + +private: + void flush(unsigned int b) { + if (buffers[b].empty()) return; + const std::string p = path(prefix, b); + FILE *f = fopen(p.c_str(), opened[b] ? "ab" : "wb"); + if (f == NULL) { + Debug(Debug::ERROR) << "Cannot open " << p << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + opened[b] = true; + if (fwrite(buffers[b].data(), sizeof(GateRow), buffers[b].size(), f) + != buffers[b].size()) { + Debug(Debug::ERROR) << "Cannot write " << p << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + if (fclose(f) != 0) { + Debug(Debug::ERROR) << "Cannot close " << p << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + buffers[b].clear(); + } + + std::string prefix; + unsigned int buckets; + size_t rowsPerBuffer; + std::vector > buffers; + std::vector opened; + bool closed; +}; + +bool byThenMember(const GateRow &a, const GateRow &b) { + if (a.sub != b.sub) return a.sub < b.sub; + return a.member < b.member; +} + +// Offset of the first byte after the newline at or after `from`, i.e. where the +// first whole line starting at or after `from` begins. Same record-boundary rule +// createdbparallel uses to make byte ranges of a text file independent. +uint64_t nextLineStart(FILE *file, uint64_t from, uint64_t size) { + if (from == 0 || from >= size) { + return std::min(from, size); + } + if (fseeko(file, static_cast(from - 1), SEEK_SET) != 0) { + Debug(Debug::ERROR) << "Cannot seek the clustering: " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + int c; + uint64_t at = from - 1; + while ((c = getc(file)) != EOF) { + at++; + if (c == '\n') { + return at; + } + } + return size; +} + +// Builds the CSR described above. One threaded pass over clu1.tsv into buckets of +// sub-key space, then one ascending sweep that emits each bucket's members +// contiguously and fills its slice of the offsets file. +// +// Resident memory is one bucket: its rows (16 B each) plus its offsets slice +// (8 B per sub-key in the range). The bucket count is derived from +// --split-memory-limit against a conservative upper bound on the row count, so a +// pathological clustering makes more buckets rather than a bigger one. +void buildFilterGate(const std::string &clusterTsv, const std::string &repDb, const Bitmap &isRep, + uint64_t entryCount, uint64_t repCount, int threads, size_t memoryLimit) { + const std::string offsetsPath = repDb + ".gate.offsets"; + const std::string membersPath = repDb + ".gate.members"; + const std::string infoPath = repDb + ".gate.info"; + const std::string spillPrefix = repDb + ".gate.spill"; + + const uint64_t tsvSize = FileUtil::getFileSize(clusterTsv); + // A row is "0\t0\n" at the very least, so this cannot under-count. + const uint64_t rowsUpperBound = tsvSize / 4 + 1; + // A floor only so a tiny limit does not explode into pathologically many + // buckets; low enough that the multi-bucket path is reachable in testing, + // which matters because that is the path used at scale. Same rule as + // mergeclusterparallel and translatecluster. + const uint64_t targetBytes = + std::max(Util::computeMemory(memoryLimit) / 8, 1ULL * 1024 * 1024); + unsigned int buckets = 1; + while (buckets < 65536 + && ((rowsUpperBound / buckets) * sizeof(GateRow) + (repCount / buckets) * sizeof(uint64_t)) + > targetBytes) { + buckets *= 2; + } + const uint64_t span = (repCount + buckets - 1) / buckets; + Debug(Debug::INFO) << "Building the pass-2 filter gate over " << buckets << " sub-key buckets " + << "of " << span << "\n"; + + // Clear anything an interrupted attempt left: the sweep below reads every + // shard it finds, so a stale one would be merged into the new gate. + for (int t = 0; t < std::max(1, threads); t++) { + for (unsigned int b = 0; b < buckets; b++) { + const std::string p = GateSpill::path(spillPrefix + ".t" + SSTR(t), b); + if (FileUtil::fileExists(p.c_str())) { + FileUtil::remove(p.c_str()); + } + } + } + + const int realThreads = std::max(1, threads); + { + std::vector writers(realThreads, NULL); + // Derived, not fixed. There are buckets x threads buffers alive at once, + // so a fixed 4096 rows each is 64 KB x buckets x threads -- 8 GB at 1024 + // buckets and 128 threads, which is the configuration this exists for. The + // budget is split across all of them, with a floor so a large bucket count + // degrades to more frequent flushes rather than to one write per row. + const size_t rowsPerBuffer = std::max( + targetBytes / (static_cast(buckets) * static_cast(realThreads) + * sizeof(GateRow)), + 64); + for (int t = 0; t < realThreads; t++) { + writers[t] = new GateSpill(spillPrefix + ".t" + SSTR(t), buckets, rowsPerBuffer); + } + const uint64_t rangeSize = (tsvSize + realThreads - 1) / realThreads; +#pragma omp parallel num_threads(realThreads) + { + int tid = 0; +#ifdef OPENMP + tid = omp_get_thread_num(); +#endif + const uint64_t nominalFrom = static_cast(tid) * rangeSize; + if (nominalFrom < tsvSize) { + const uint64_t nominalTo = std::min(nominalFrom + rangeSize, tsvSize); + FILE *f = FileUtil::openFileOrDie(clusterTsv.c_str(), "r", true); + const uint64_t from = nextLineStart(f, nominalFrom, tsvSize); + if (fseeko(f, static_cast(from), SEEK_SET) != 0) { + Debug(Debug::ERROR) << "Cannot seek " << clusterTsv << ": " << strerror(errno) + << "\n"; + EXIT(EXIT_FAILURE); + } + char *line = NULL; + size_t cap = 0; + ssize_t len; + uint64_t at = from; + while (at < nominalTo && (len = getline(&line, &cap, f)) > 0) { + at += static_cast(len); + char *tab = strchr(line, '\t'); + if (tab == NULL) continue; + const uint64_t rep = strtoull(line, NULL, 10); + const uint64_t member = strtoull(tab + 1, NULL, 10); + if (rep >= entryCount || member >= entryCount) { + Debug(Debug::ERROR) << "Clustering " << clusterTsv << " names key " + << std::max(rep, member) << ", beyond the " + << entryCount << " in the database\n"; + EXIT(EXIT_FAILURE); + } + // Every first-column key is a representative by construction, + // so this rank is its sub-key. + const uint64_t sub = isRep.rankOf(rep); + writers[tid]->append(static_cast(sub / span), sub, member); + } + free(line); + fclose(f); + } + } + for (int t = 0; t < realThreads; t++) { + delete writers[t]; + } + } + + // Sweep the buckets in ascending sub-key order, so members are emitted + // contiguously and the running total *is* the offset. + const std::string offsetsTmp = offsetsPath + ".tmp"; + const std::string membersTmp = membersPath + ".tmp"; + FILE *offsetsOut = FileUtil::openAndDelete(offsetsTmp.c_str(), "wb"); + FILE *membersOut = FileUtil::openAndDelete(membersTmp.c_str(), "wb"); + std::vector rows; + std::vector offsets; + std::vector members; + uint64_t total = 0; + for (unsigned int b = 0; b < buckets; b++) { + const uint64_t lo = static_cast(b) * span; + if (lo >= repCount) break; + const uint64_t hi = std::min(lo + span, repCount); + + rows.clear(); + for (int t = 0; t < realThreads; t++) { + const std::string shardPrefix = spillPrefix + ".t" + SSTR(t); + GateSpill::read(shardPrefix, b, rows); + const std::string p = GateSpill::path(shardPrefix, b); + if (FileUtil::fileExists(p.c_str())) { + FileUtil::remove(p.c_str()); + } + } + // By (sub, member): grouping is what the CSR needs, and the second key + // makes the file a function of the clustering alone rather than of which + // thread happened to read which byte range. + SORT_SERIAL(rows.begin(), rows.end(), byThenMember); + + offsets.assign(static_cast(hi - lo), 0); + members.clear(); + members.reserve(rows.size()); + size_t i = 0; + for (uint64_t sub = lo; sub < hi; sub++) { + offsets[static_cast(sub - lo)] = total + members.size(); + while (i < rows.size() && rows[i].sub == sub) { + members.push_back(rows[i].member); + i++; + } + } + if (i != rows.size()) { + Debug(Debug::ERROR) << "Gate bucket " << b << " holds sub-key " << rows[i].sub + << ", outside its range [" << lo << ", " << hi << ")\n"; + EXIT(EXIT_FAILURE); + } + if (offsets.empty() == false + && fwrite(offsets.data(), sizeof(uint64_t), offsets.size(), offsetsOut) + != offsets.size()) { + Debug(Debug::ERROR) << "Cannot write " << offsetsTmp << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + if (members.empty() == false + && fwrite(members.data(), sizeof(uint64_t), members.size(), membersOut) + != members.size()) { + Debug(Debug::ERROR) << "Cannot write " << membersTmp << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + total += members.size(); + } + // The closing sentinel, so offsets[s + 1] is always readable. + if (fwrite(&total, sizeof(uint64_t), 1, offsetsOut) != 1) { + Debug(Debug::ERROR) << "Cannot write " << offsetsTmp << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + if (fclose(offsetsOut) != 0 || fclose(membersOut) != 0) { + Debug(Debug::ERROR) << "Cannot close the filter gate files: " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + if (FileUtil::getFileSize(offsetsTmp) != (repCount + 1) * sizeof(uint64_t)) { + Debug(Debug::ERROR) << offsetsTmp << " holds " + << (FileUtil::getFileSize(offsetsTmp) / sizeof(uint64_t)) + << " offsets, not the " << (repCount + 1) << " expected\n"; + EXIT(EXIT_FAILURE); + } + FileUtil::move(membersTmp.c_str(), membersPath.c_str()); + FileUtil::move(offsetsTmp.c_str(), offsetsPath.c_str()); + + const std::string infoTmp = infoPath + ".tmp"; + FILE *infoOut = FileUtil::openAndDelete(infoTmp.c_str(), "w"); + fprintf(infoOut, "repCount\t%zu\n", (size_t)repCount); + fprintf(infoOut, "memberCount\t%zu\n", (size_t)total); + if (fclose(infoOut) != 0) { + Debug(Debug::ERROR) << "Cannot close " << infoTmp << "\n"; + EXIT(EXIT_FAILURE); + } + FileUtil::move(infoTmp.c_str(), infoPath.c_str()); + + Debug(Debug::INFO) << "Filter gate: " << repCount << " clusters, " << total << " members (" + << (total * sizeof(uint64_t)) / (1024 * 1024) << " MB on disk, paged by " + << "the align workers rather than held)\n"; +} + } // namespace int createrepdb(int argc, const char **argv, const Command &command) { @@ -228,23 +737,62 @@ int createrepdb(int argc, const char **argv, const Command &command) { const std::string repDb = par.db3; const DenseIndex::Info info = DenseIndex::readInfo(seqDb); + if (info.entryCount == 0) { + Debug(Debug::ERROR) << "Database " << seqDb << " is empty\n"; + EXIT(EXIT_FAILURE); + } // A key is a representative exactly when it appears in the first column. + // + // Read by byte range across threads rather than one getline loop. At 1e12 this + // file is ~28 TB and the scan was the serial head of a stage that is otherwise + // threaded. Ranges are made independent the same way createdbparallel makes + // FASTA chunks independent: a range owns the lines that *start* inside it, so + // each thread skips to the first line boundary at or after its start and reads + // through the boundary at or after its end. Bitmap isRep(info.entryCount); uint64_t repCount = 0; { - FILE *f = FileUtil::openFileOrDie(clusterTsv.c_str(), "r", true); - char *line = NULL; - size_t cap = 0; - while (getline(&line, &cap, f) > 0) { - const uint64_t rep = strtoull(line, NULL, 10); - if (rep < info.entryCount && isRep.get(rep) == false) { - isRep.set(rep); - repCount++; + const uint64_t tsvSize = FileUtil::getFileSize(clusterTsv); + if (tsvSize == 0) { + Debug(Debug::ERROR) << "Clustering " << clusterTsv << " is empty\n"; + EXIT(EXIT_FAILURE); + } + const int threads = std::max(1, par.threads); + const uint64_t rangeSize = (tsvSize + threads - 1) / threads; + uint64_t counted = 0; +#pragma omp parallel num_threads(threads) reduction(+ : counted) + { + int tid = 0; +#ifdef OPENMP + tid = omp_get_thread_num(); +#endif + const uint64_t nominalFrom = static_cast(tid) * rangeSize; + if (nominalFrom < tsvSize) { + const uint64_t nominalTo = std::min(nominalFrom + rangeSize, tsvSize); + FILE *f = FileUtil::openFileOrDie(clusterTsv.c_str(), "r", true); + const uint64_t from = nextLineStart(f, nominalFrom, tsvSize); + if (fseeko(f, static_cast(from), SEEK_SET) != 0) { + Debug(Debug::ERROR) << "Cannot seek " << clusterTsv << ": " << strerror(errno) + << "\n"; + EXIT(EXIT_FAILURE); + } + char *line = NULL; + size_t cap = 0; + ssize_t len; + uint64_t at = from; + while (at < nominalTo && (len = getline(&line, &cap, f)) > 0) { + at += static_cast(len); + const uint64_t rep = strtoull(line, NULL, 10); + if (rep < info.entryCount && isRep.testAndSet(rep)) { + counted++; + } + } + free(line); + fclose(f); } } - free(line); - fclose(f); + repCount = counted; } Debug(Debug::INFO) << repCount << " representatives of " << info.entryCount << " sequences (" << isRep.bytes() / (1024 * 1024) << " MB of flags)\n"; @@ -253,28 +801,37 @@ int createrepdb(int argc, const char **argv, const Command &command) { EXIT(EXIT_FAILURE); } - std::vector keyMap(repCount, 0); - const CopyResult seq = copyFlagged(seqDb, repDb, isRep, info.entryCount, repCount, par.threads, - &keyMap); - copyFlagged(seqDb + "_h", repDb + "_h", isRep, info.entryCount, repCount, par.threads, NULL); - // subkey -> original key, dense in the sub-key space and in ascending order, // so the translation back is a sequential read rather than a lookup structure. + // Streamed out by the sequence copy's plan pass, which is the one place that + // already visits every representative in order. const std::string mapFile = repDb + ".keymap"; const std::string keymapTmp = mapFile + ".tmp"; FILE *m = FileUtil::openAndDelete(keymapTmp.c_str(), "wb"); - if (fwrite(keyMap.data(), sizeof(uint64_t), keyMap.size(), m) != keyMap.size()) { - Debug(Debug::ERROR) << "Cannot write " << keymapTmp << "\n"; - EXIT(EXIT_FAILURE); - } - if (fclose(m) != 0) { - Debug(Debug::ERROR) << "Cannot close " << keymapTmp << ": " << strerror(errno) << "\n"; + const CopyResult seq = + copyFlagged(seqDb, repDb, isRep, info.entryCount, repCount, par.threads, m, keymapTmp); + if (fclose(m) != 0) { + Debug(Debug::ERROR) << "Cannot close " << keymapTmp << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + if (FileUtil::getFileSize(keymapTmp) != repCount * sizeof(uint64_t)) { + Debug(Debug::ERROR) << keymapTmp << " holds " + << (FileUtil::getFileSize(keymapTmp) / sizeof(uint64_t)) + << " keys, not the " << repCount << " representatives copied\n"; EXIT(EXIT_FAILURE); - } + copyFlagged(seqDb + "_h", repDb + "_h", isRep, info.entryCount, repCount, par.threads, NULL, ""); + + // The pass-2 filter gate, built once here instead of in every align worker. + // Needs the rank directory, which turns an original key into its sub-key. + isRep.buildRank(); + Debug(Debug::INFO) << "Rank directory: " << isRep.rankBytes() / (1024 * 1024) << " MB\n"; + buildFilterGate(clusterTsv, repDb, isRep, info.entryCount, repCount, par.threads, + par.splitMemoryLimit); + const int dbType = FileUtil::parseDbType(seqDb.c_str()); FileUtil::writeFile(repDb + ".dbtype", reinterpret_cast(&dbType), sizeof(int)); diff --git a/src/linclust/greedycluster.cpp b/src/linclust/greedycluster.cpp index 58c1c6114..5fc555ecd 100644 --- a/src/linclust/greedycluster.cpp +++ b/src/linclust/greedycluster.cpp @@ -42,10 +42,12 @@ #include "DenseIndex.h" #include "FastSort.h" #include "FileUtil.h" +#include "ParallelCoordination.h" #include "Parameters.h" #include "Util.h" #include +#include #include #include #include @@ -146,7 +148,15 @@ size_t readBucket(const std::string &alnDir, unsigned int bucket, std::vector(out.data()); - bool failed = false; + // Both atomic. Threads used to write a plain `bool failed` concurrently, which + // is a data race and so undefined behaviour -- in practice the main thread + // could miss the flag entirely and treat a short or failed read as a complete + // bucket, dropping edges with no diagnostic. The error code is captured by the + // failing thread too, because reporting the *calling* thread's errno after the + // region names whatever happened to fail last in that thread, which is usually + // nothing. + std::atomic failed(false); + std::atomic failedErrno(0); #pragma omp parallel for schedule(dynamic, 1) num_threads(threads) for (size_t c = 0; c < chunks; c++) { const size_t from = c * chunkBytes; @@ -159,15 +169,18 @@ size_t readBucket(const std::string &alnDir, unsigned int bucket, std::vector(got); } } close(fd); - if (failed) { - Debug(Debug::ERROR) << "Cannot read " << path << ": " << strerror(errno) << "\n"; + if (failed.load()) { + Debug(Debug::ERROR) << "Cannot read " << path << ": " << strerror(failedErrno.load()) + << "\n"; EXIT(EXIT_FAILURE); } return out.size(); @@ -186,17 +199,75 @@ int greedycluster(int argc, const char **argv, const Command &command) { const DenseIndex::Info info = DenseIndex::readInfo(seqDb); par.printParameters(command.cmd, argc, argv, *command.params); - // Bucket count comes from the align stage's layout: one file per bucket, and - // buckets are ascending representative-key ranges, which is exactly the order - // the sweep needs. + // Bucket count and span come from the layout alignparallel recorded, and the + // align queue is checked complete before anything is read. + // + // Counting consecutive p.edges files instead -- which is what this used to + // do -- treats a *prefix* of the buckets as the whole layout. An align stage + // that had produced only the first few buckets therefore looked complete, and + // bucketSpan was recomputed from the smaller count, so the sweep walked key + // ranges that did not correspond to any bucket. The result is a successful, + // silently wrong clustering, and a restart preserves it because the greedy + // output now exists. + const std::string layoutPath = alnDir + "/coord/align.info"; + if (FileUtil::fileExists(layoutPath.c_str()) == false) { + Debug(Debug::ERROR) << "No align layout at " << layoutPath + << ". Run alignparallel first (with this build: older ones did not " + << "record the layout).\n"; + EXIT(EXIT_FAILURE); + } unsigned int bucketCount = 0; - while (FileUtil::fileExists(EdgeWriter::partitionPath(alnDir, bucketCount).c_str())) { - bucketCount++; + uint64_t bucketSpan = 0; + uint64_t layoutEntryCount = 0; + { + FILE *f = FileUtil::openFileOrDie(layoutPath.c_str(), "r", true); + char name[64]; + size_t value; + while (fscanf(f, "%63s\t%zu\n", name, &value) == 2) { + const std::string key = name; + if (key == "bucketCount") bucketCount = static_cast(value); + else if (key == "bucketSpan") bucketSpan = value; + else if (key == "entryCount") layoutEntryCount = value; + } + fclose(f); } - if (bucketCount == 0) { - Debug(Debug::ERROR) << "No edge buckets in " << alnDir << ". Run alignparallel first.\n"; + if (bucketCount == 0 || bucketSpan == 0) { + Debug(Debug::ERROR) << "Align layout " << layoutPath << " is incomplete\n"; EXIT(EXIT_FAILURE); } + if (layoutEntryCount != info.entryCount) { + Debug(Debug::ERROR) << "The alignments in " << alnDir << " were produced for " + << layoutEntryCount << " sequences, but " << seqDb << " holds " + << info.entryCount << ". They are from different runs.\n"; + EXIT(EXIT_FAILURE); + } + { + std::vector workers; + if (WorkQueue::readCompletedWorkers(alnDir + "/coord/align.queue", workers) == false) { + Debug(Debug::ERROR) << "No align work queue in " << alnDir + << "/coord. Run alignparallel first.\n"; + EXIT(EXIT_FAILURE); + } + if (workers.size() != bucketCount) { + Debug(Debug::ERROR) << "The align queue covers " << workers.size() + << " buckets but the layout records " << bucketCount << "\n"; + EXIT(EXIT_FAILURE); + } + for (size_t i = 0; i < workers.size(); i++) { + if (workers[i] < 0) { + Debug(Debug::ERROR) << "Bucket " << i << " was never aligned. Re-run " + << "alignparallel before clustering.\n"; + EXIT(EXIT_FAILURE); + } + } + } + for (unsigned int b = 0; b < bucketCount; b++) { + if (FileUtil::fileExists(EdgeWriter::partitionPath(alnDir, b).c_str()) == false) { + Debug(Debug::ERROR) << "Bucket " << b << " is recorded aligned but " + << EdgeWriter::partitionPath(alnDir, b) << " is missing\n"; + EXIT(EXIT_FAILURE); + } + } KeyFlags flags(info.entryCount); Debug(Debug::INFO) << "Sweeping " << info.entryCount << " keys over " << bucketCount @@ -212,8 +283,8 @@ int greedycluster(int argc, const char **argv, const Command &command) { // Edge buckets are contiguous ascending representative-key ranges, the same // ranges kmerreduceparallel derived, so walking buckets in order walks the key - // space in order. - const uint64_t bucketSpan = (info.entryCount + bucketCount - 1) / bucketCount; + // space in order. bucketSpan comes from the layout above rather than being + // re-derived, so it cannot disagree with the ranges the edges were bucketed on. for (unsigned int bucket = 0; bucket < bucketCount; bucket++) { const uint64_t lo = bucket * bucketSpan; @@ -222,8 +293,23 @@ int greedycluster(int argc, const char **argv, const Command &command) { } const uint64_t hi = std::min(lo + bucketSpan, info.entryCount); readBucket(alnDir, bucket, edges, par.threads); - if (edges.empty() == false) { - SORT_PARALLEL(edges.begin(), edges.end(), compareByRepThenMember); + // Verified, not sorted. alignparallel produces this file already in + // (rep, member) order: mergePairCopies sorts by (rep, member, diagonal, + // strand), compacts to one record per pair in place, and the survivors are + // then appended in index order into one file per bucket, published by + // atomic rename. Re-sorting up to ~74 GB of edges per bucket at 1e12 was + // therefore pure cost -- but the sweep below is silently wrong if the order + // ever does not hold, so the assumption is checked rather than assumed. A + // linear scan against an O(n log n) sort is a trade worth making in both + // directions. + if (edges.empty() == false + && std::is_sorted(edges.begin(), edges.end(), compareByRepThenMember) == false) { + Debug(Debug::ERROR) << "Edge bucket " << EdgeWriter::partitionPath(alnDir, bucket) + << " is not ordered by (representative, member). The greedy sweep " + << "consumes it in one forward pass and would silently drop the " + << "out-of-order edges. It was not written by this build's " + << "alignparallel.\n"; + EXIT(EXIT_FAILURE); } size_t p = 0; diff --git a/src/linclust/kmermatcherparallel.cpp b/src/linclust/kmermatcherparallel.cpp index 055523c6a..a3b43f437 100644 --- a/src/linclust/kmermatcherparallel.cpp +++ b/src/linclust/kmermatcherparallel.cpp @@ -36,11 +36,15 @@ #include "Util.h" #include "kmermatcher.h" +#include #include #include +#include #include #include +#include + namespace { // Every file a sequence database actually occupies, not just its data file. @@ -99,15 +103,49 @@ struct ShuffleManifest { uint64_t partitionCount; uint64_t waveCount; uint64_t kmerSize; + // Carried for the same reason as kmerSize, and it is the same class of bug. + // + // setKmerLengthAndAlphabet picks *both* from --min-seq-id when -k is 0 + // (kmermatcher.cpp:2098-2110): 14/21 at >= 0.99, 14/13 at >= 0.9. The map runs + // it; the reduce cannot, because it has no sequence database to size it from, + // so it used to take kmerSize from here and leave the alphabet at + // setLinearFilterDefault's CLUST_LINEAR_DEFAULT_ALPH_SIZE of 13. + // + // At >= 0.99 that is a real disagreement: the map encodes + // KmerRecord::adjacent[] with the 21-letter aa2num, and the reduce would score + // those codes through a ReducedMatrix(13) whose rows 13..20 are never filled by + // generateSubMatrix. assignGroup's adjacency rounds consume exactly that, so + // the re-centring is scored against zeros. At 0.9 the two agreed only because + // both independently landed on 13. + // + // Recording it removes the whole class: anything the map derives that the + // reduce needs belongs in the manifest, not in a flag both sides must be + // passed identically. + uint64_t alphabetSizeAa; + uint64_t alphabetSizeNucl; void write(const std::string &path) const { - FILE *file = FileUtil::openAndDelete(path.c_str(), "w"); + // Written to a private temporary and renamed, never opened at the + // authoritative path. Existence of shuffle.info is what every later worker + // takes as proof the shuffle was laid out, so a worker killed between the + // open (which truncates) and the close left an empty or half-written + // manifest that all of them then read as authoritative -- and no restart + // could repair it, because the path existed. rename(2) is atomic, so the + // file either is not there or is complete. + const std::string tmp = path + ".tmp." + SSTR(getpid()); + FILE *file = FileUtil::openAndDelete(tmp.c_str(), "w"); fprintf(file, "entryCount\t%zu\n", (size_t)entryCount); fprintf(file, "partitionCount\t%zu\n", (size_t)partitionCount); fprintf(file, "waveCount\t%zu\n", (size_t)waveCount); fprintf(file, "kmerSize\t%zu\n", (size_t)kmerSize); + fprintf(file, "alphabetSizeAa\t%zu\n", (size_t)alphabetSizeAa); + fprintf(file, "alphabetSizeNucl\t%zu\n", (size_t)alphabetSizeNucl); if (fclose(file) != 0) { - Debug(Debug::ERROR) << "Cannot close " << path << "\n"; + Debug(Debug::ERROR) << "Cannot close " << tmp << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + if (rename(tmp.c_str(), path.c_str()) != 0) { + Debug(Debug::ERROR) << "Cannot publish " << path << ": " << strerror(errno) << "\n"; EXIT(EXIT_FAILURE); } } @@ -121,6 +159,8 @@ struct ShuffleManifest { manifest.partitionCount = 0; manifest.waveCount = 0; manifest.kmerSize = 0; + manifest.alphabetSizeAa = 0; + manifest.alphabetSizeNucl = 0; while (fscanf(file, "%63s\t%zu\n", name, &value) == 2) { const std::string key = name; if (key == "entryCount") { @@ -131,6 +171,10 @@ struct ShuffleManifest { manifest.waveCount = value; } else if (key == "kmerSize") { manifest.kmerSize = value; + } else if (key == "alphabetSizeAa") { + manifest.alphabetSizeAa = value; + } else if (key == "alphabetSizeNucl") { + manifest.alphabetSizeNucl = value; } } fclose(file); @@ -312,12 +356,16 @@ int kmermatcherparallel(int argc, const char **argv, const Command &command) { // filesystem: a different database or k-mer length means the workers // are not running the same job at all. if (existing.entryCount != info.entryCount || - existing.kmerSize != static_cast(par.kmerSize)) { + existing.kmerSize != static_cast(par.kmerSize) || + existing.alphabetSizeAa != + static_cast(par.alphabetSize.values.aminoacid())) { Debug(Debug::ERROR) << "The shuffle already in progress (" << manifestPath << ") covers " << existing.entryCount << " sequences at k=" << existing.kmerSize + << ", alphabet " << existing.alphabetSizeAa << ", but this worker was given " << info.entryCount << " sequences at k=" - << par.kmerSize << ".\n" + << par.kmerSize << ", alphabet " << par.alphabetSize.values.aminoacid() + << ".\n" << "Every worker must run the same command line on the same database.\n"; EXIT(EXIT_FAILURE); } @@ -333,8 +381,15 @@ int kmermatcherparallel(int argc, const char **argv, const Command &command) { sizing.waveCount = static_cast(existing.waveCount); sizing.bytesPerWave = (sizing.totalKmerBytes + sizing.waveCount - 1) / sizing.waveCount; + // From totalKmerBytes, not bytesPerWave -- the same distinction + // KmerPartition.cpp:108-120 spells out for the derivation path. A wave + // keeps *whole* partitions, so a partition always holds + // totalKmerBytes / P however many waves there are; dividing the wave's + // share again reported a figure waveCount times too small. Only a + // diagnostic today, but it is the number anything sizing itself + // against a partition would reach for. sizing.bytesPerPartition = - (sizing.bytesPerWave + sizing.partitionCount - 1) / sizing.partitionCount; + (sizing.totalKmerBytes + sizing.partitionCount - 1) / sizing.partitionCount; } else { sizing = deriveKmerShuffleSizing(info.entryCount, kmersPerSequence, par.scratchBudget, persistentBytes, @@ -344,6 +399,12 @@ int kmermatcherparallel(int argc, const char **argv, const Command &command) { manifest.partitionCount = sizing.partitionCount; manifest.waveCount = sizing.waveCount; manifest.kmerSize = static_cast(par.kmerSize); + // Recorded after setKmerLengthAndAlphabet has run, so this is the + // alphabet the k-mers in the buckets were actually encoded with. + manifest.alphabetSizeAa = + static_cast(par.alphabetSize.values.aminoacid()); + manifest.alphabetSizeNucl = + static_cast(par.alphabetSize.values.nucleotide()); KmerBucketWriter::createLayout(kmerDir, sizing.partitionCount); manifest.write(manifestPath); } diff --git a/src/linclust/kmerreduceparallel.cpp b/src/linclust/kmerreduceparallel.cpp index daba301b0..e814abbdf 100644 --- a/src/linclust/kmerreduceparallel.cpp +++ b/src/linclust/kmerreduceparallel.cpp @@ -388,6 +388,8 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { unsigned int partitionCount = 0; unsigned int kmerSize = 0; unsigned int waveCount = 1; + unsigned int alphabetSizeAa = 0; + unsigned int alphabetSizeNucl = 0; { FILE *file = FileUtil::openFileOrDie(manifestPath.c_str(), "r", true); char name[64]; @@ -400,6 +402,10 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { kmerSize = static_cast(value); } else if (key == "waveCount") { waveCount = static_cast(value); + } else if (key == "alphabetSizeAa") { + alphabetSizeAa = static_cast(value); + } else if (key == "alphabetSizeNucl") { + alphabetSizeNucl = static_cast(value); } } fclose(file); @@ -408,6 +414,13 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { Debug(Debug::ERROR) << "Shuffle manifest " << manifestPath << " has no partition count\n"; EXIT(EXIT_FAILURE); } + if (alphabetSizeAa == 0) { + Debug(Debug::ERROR) << "Shuffle manifest " << manifestPath << " has no alphabet size. It " + << "was written by an older build, whose reduce scored the k-mers with " + << "a different alphabet than the map encoded them in. Re-run the map " + << "for this shuffle with this build.\n"; + EXIT(EXIT_FAILURE); + } // A wave's map wrote only its own slice of partition space, so its reduce // claims exactly that slice. The slicing must be identical to the map's, and // each wave needs its own queue: a shared one would record the whole @@ -437,9 +450,20 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { << "written in one wave\n"; EXIT(EXIT_FAILURE); } - // The map decided k, so take it from the manifest rather than re-deriving it - // here: the two must agree, and the map's value is the one on disk. + // The map decided k *and* the alphabet, so both come from the manifest rather + // than being re-derived or re-passed here. setKmerLengthAndAlphabet picks them + // together from --min-seq-id (kmermatcher.cpp:2098-2110), and this command + // cannot run it -- it has no sequence database to size it from. Taking only k + // and leaving the alphabet at setLinearFilterDefault's 13 meant that at + // --min-seq-id >= 0.99 the map encoded adjacent residues with the 21-letter + // aa2num while the reduce scored them through a ReducedMatrix(13) whose rows + // 13..20 are zero. The alphabet is a property of the k-mers on disk, so the + // manifest is where it belongs. par.kmerSize = static_cast(kmerSize); + par.alphabetSize.values.aminoacid(static_cast(alphabetSizeAa)); + if (alphabetSizeNucl > 0) { + par.alphabetSize.values.nucleotide(static_cast(alphabetSizeNucl)); + } par.printParameters(command.cmd, argc, argv, *command.params); Debug(Debug::INFO) << "Reducing " << partitionCount << " partitions of " << kmerDir << "\n"; @@ -461,31 +485,54 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { } uint64_t bucketSpan = (info.entryCount + bucketCount - 1) / bucketCount; + // The manifest is authoritative once it exists; the derivation above only + // creates it. This is the same rule the map applies to shuffle.info, and it is + // load-bearing for the same reason: bucketCount comes from + // Util::computeMemory(--split-memory-limit), which with the workflow default + // of 0 is *this node's* physical RAM (and ignores cgroup limits besides). + // + // Refusing to run when the two disagree -- which is what this used to do -- + // meant one differently-sized node in a Slurm array, a restart on another + // partition, or a container memory limit killed the whole stage. Adopting the + // recorded layout instead is both safer and correct: the bucketing has to be + // fixed for the life of the edge directory, and the run that created it is the + // one that decided. const std::string edgeManifest = reduceCoordDir + "/edge.info"; { FileLock lock(reduceCoordDir + "/edge.lock"); lock.lock(); if (FileUtil::fileExists(edgeManifest.c_str()) == false) { EdgeBucketWriter::createLayout(edgeDir, bucketCount); - FILE *f = FileUtil::openAndDelete(edgeManifest.c_str(), "w"); + // Private temporary then rename, so a worker killed mid-write cannot + // leave an empty manifest that every later worker reads as + // authoritative and no restart can repair. + const std::string tmp = edgeManifest + ".tmp." + SSTR(getpid()); + FILE *f = FileUtil::openAndDelete(tmp.c_str(), "w"); fprintf(f, "bucketCount\t%zu\n", (size_t)bucketCount); fprintf(f, "bucketSpan\t%zu\n", (size_t)bucketSpan); fprintf(f, "entryCount\t%zu\n", (size_t)info.entryCount); - fclose(f); + // The shape of the reduce itself, so the align stage can tell "every + // partition has been reduced" from "the queues I happened to find were + // complete". Without it, starting the align before the last wave + // existed produced a short authority vector that looked finished. + fprintf(f, "partitionCount\t%zu\n", (size_t)partitionCount); + fprintf(f, "waveCount\t%zu\n", (size_t)waveCount); + if (fclose(f) != 0) { + Debug(Debug::ERROR) << "Cannot close " << tmp << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + if (rename(tmp.c_str(), edgeManifest.c_str()) != 0) { + Debug(Debug::ERROR) << "Cannot publish " << edgeManifest << ": " << strerror(errno) + << "\n"; + EXIT(EXIT_FAILURE); + } } lock.unlock(); } - // Re-read what the manifest actually says and refuse to disagree with it. - // - // Every worker derives bucketCount from Util::computeMemory(--split-memory-limit), - // which with the workflow's default of 0 is *this node's* RAM. A heterogeneous - // Slurm array, a restart on a different node, or a later wave would otherwise - // route edges into a different bucketing than the run began with -- writing - // into r directories createLayout never made. The map already does exactly - // this for shuffle.info. { unsigned int fileBucketCount = 0; uint64_t fileBucketSpan = 0; + uint64_t fileEntryCount = 0; FILE *f = FileUtil::openFileOrDie(edgeManifest.c_str(), "r", true); char name[64]; size_t value; @@ -493,18 +540,31 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { const std::string key = name; if (key == "bucketCount") fileBucketCount = static_cast(value); else if (key == "bucketSpan") fileBucketSpan = value; + else if (key == "entryCount") fileEntryCount = value; } fclose(f); - if (fileBucketCount != bucketCount || fileBucketSpan != bucketSpan) { - Debug(Debug::ERROR) << "This worker derived " << bucketCount << " edge buckets of " - << bucketSpan << " keys, but " << edgeManifest << " says " - << fileBucketCount << " of " << fileBucketSpan - << ". The run was started with a different --split-memory-limit or " - << "on a node with different memory; edges would be routed into a " - << "different bucketing than the rest of the run. Re-run every " - << "worker of this stage with the same --split-memory-limit.\n"; + if (fileBucketCount == 0 || fileBucketSpan == 0) { + Debug(Debug::ERROR) << "Edge manifest " << edgeManifest << " has no bucket layout\n"; EXIT(EXIT_FAILURE); } + // entryCount still *is* checked, because unlike the bucketing it describes + // the input rather than the node: a different database means these workers + // are not running the same job at all, and the key ranges would not line up. + if (fileEntryCount != info.entryCount) { + Debug(Debug::ERROR) << "The edge buckets at " << edgeDir << " were laid out for " + << fileEntryCount << " sequences, but this worker was given " + << info.entryCount << ". Every worker must run on the same " + << "database.\n"; + EXIT(EXIT_FAILURE); + } + if (fileBucketCount != bucketCount || fileBucketSpan != bucketSpan) { + Debug(Debug::WARNING) << "This worker would have derived " << bucketCount + << " edge buckets of " << bucketSpan << " keys from its own " + << "memory, but " << edgeManifest << " says " << fileBucketCount + << " of " << fileBucketSpan + << "; using the recorded layout. This is expected on a " + << "heterogeneous allocation or under a cgroup memory limit.\n"; + } bucketCount = fileBucketCount; bucketSpan = fileBucketSpan; } @@ -561,28 +621,55 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { // intermediate the pipeline writes. // // Safe here because drain() returns only once every partition of the wave - // is recorded done, so no worker holding a live lease is still reading - // one. Every worker that observes the queue finished reaches this and - // unlinks, hence the ENOENT tolerance below. + // is recorded done, so no worker holding a live lease is still reading one + // -- and, just as importantly, so that a worker cannot delete an item's + // input before the item is recorded complete. Deleting inside the work + // item would be cheaper still, but a worker that unlinked and then died + // before completing would leave the redo to read an empty partition, emit + // no edges, and record *that* as success. + // + // Done by exactly one worker, under a lock and behind a sentinel. Every + // worker used to walk every partition of the wave and unlink every shard: + // at 1e12 that is 8192 workers x 1024 partitions x 8192 shards, roughly + // 7e10 unlink syscalls of which all but 1/8192 return ENOENT, plus 8.4e6 + // opendir calls per worker. The others block only for as long as the one + // sweep takes, which is work they were all doing anyway. + // + // The sentinel rather than a counter, so an interrupted sweep is redone by + // the next run of the stage instead of being remembered as finished. // // A worker whose lease lapsed while still inside reducePartition can be - // reading a shard as another unlinks it. That costs nothing: its item was - // redone and recorded by someone else, so its edges are discarded by block - // header anyway. What it must not do is turn into a stage failure, so + // reading a shard as the sweeper unlinks it. That costs nothing: its item + // was redone and recorded by someone else, so its edges are discarded by + // block header anyway. What it must not do is turn into a stage failure, so // readPartitionAsPositions treats a shard that disappears under it as // empty rather than as an error. - for (unsigned int p = waveFrom; p < waveTo; p++) { - const std::vector shards = KmerBucketReader::shardFiles(kmerDir, p); - for (size_t i = 0; i < shards.size(); i++) { - if (unlink(shards[i].c_str()) != 0 && errno != ENOENT) { - Debug(Debug::ERROR) << "Cannot remove reduced bucket " << shards[i] << ": " - << strerror(errno) << "\n"; - EXIT(EXIT_FAILURE); + const std::string waveTag = SSTR(par.kmerWave < 0 ? 0 : par.kmerWave); + const std::string cleanupDone = reduceCoordDir + "/cleanup." + waveTag + ".done"; + FileLock cleanupLock(reduceCoordDir + "/cleanup." + waveTag + ".lock"); + cleanupLock.lock(); + if (FileUtil::fileExists(cleanupDone.c_str()) == false) { + for (unsigned int p = waveFrom; p < waveTo; p++) { + const std::vector shards = KmerBucketReader::shardFiles(kmerDir, p); + for (size_t i = 0; i < shards.size(); i++) { + if (unlink(shards[i].c_str()) != 0 && errno != ENOENT) { + Debug(Debug::ERROR) << "Cannot remove reduced bucket " << shards[i] << ": " + << strerror(errno) << "\n"; + cleanupLock.unlock(); + EXIT(EXIT_FAILURE); + } } } + FILE *sentinel = FileUtil::openAndDelete(cleanupDone.c_str(), "w"); + if (fclose(sentinel) != 0) { + Debug(Debug::ERROR) << "Cannot close " << cleanupDone << "\n"; + cleanupLock.unlock(); + EXIT(EXIT_FAILURE); + } + Debug(Debug::INFO) << "Removed the consumed k-mer buckets" + << (waveCount > 1 ? " of wave " + SSTR(par.kmerWave) : "") << "\n"; } - Debug(Debug::INFO) << "Removed the consumed k-mer buckets" - << (waveCount > 1 ? " of wave " + SSTR(par.kmerWave) : "") << "\n"; + cleanupLock.unlock(); } Debug(Debug::INFO) << "Worker " << workerId << " wrote " << edgeCount diff --git a/src/linclust/mergeclusterparallel.cpp b/src/linclust/mergeclusterparallel.cpp index 6d9802579..49af2d8cc 100644 --- a/src/linclust/mergeclusterparallel.cpp +++ b/src/linclust/mergeclusterparallel.cpp @@ -131,7 +131,15 @@ class BucketWriter { }; // Streams a "repmember" TSV, bucketing on whichever column the join needs. -void partition(const std::string &tsv, BucketWriter &writer, uint64_t bucketSpan, bool byRep) { +// +// Both columns are checked against the key space *before* either is divided into a +// bucket index. `key / bucketSpan` indexes the writer's bucket vector immediately, +// and the value ends up indexing `remap[key - lo]` in compose(), so a key from a +// clustering built over a different database wrote out of bounds long before +// anything downstream could report it. Dense keys make the valid range exactly +// [0, entryCount). +void partition(const std::string &tsv, BucketWriter &writer, uint64_t bucketSpan, bool byRep, + uint64_t entryCount) { FILE *file = FileUtil::openFileOrDie(tsv.c_str(), "r", true); char *line = NULL; size_t cap = 0; @@ -143,6 +151,13 @@ void partition(const std::string &tsv, BucketWriter &writer, uint64_t bucketSpan } const uint64_t rep = strtoull(line, NULL, 10); const uint64_t member = strtoull(tab + 1, NULL, 10); + if (rep >= entryCount || member >= entryCount) { + Debug(Debug::ERROR) << "Clustering " << tsv << " names key " + << std::max(rep, member) << ", beyond the " << entryCount + << " keys of the database. The clusterings being merged and the " + << "database are from different runs.\n"; + EXIT(EXIT_FAILURE); + } const uint64_t key = byRep ? rep : member; const uint64_t value = byRep ? member : rep; writer.append(static_cast(key / bucketSpan), key, value); @@ -205,8 +220,8 @@ void compose(const std::string &earlier, const std::string &later, const std::st // Bucket the later clustering by the member it reassigns, and the earlier // one by its representative: those are the two sides of the join, so they // meet in the same bucket. - partition(later, laterW, bucketSpan, false); - partition(earlier, earlierW, bucketSpan, true); + partition(later, laterW, bucketSpan, false, entryCount); + partition(earlier, earlierW, bucketSpan, true, entryCount); } FILE *result = FileUtil::openAndDelete(out.c_str(), "w"); @@ -226,12 +241,30 @@ void compose(const std::string &earlier, const std::string &later, const std::st remap.assign(static_cast(hi - lo), INVALID); const std::vector laterPairs = readBucket(tmpPrefix + ".later", b); for (size_t i = 0; i < laterPairs.size(); i++) { + // Re-checked on the way out of the spill file, not only on the way in: + // this is the index into `remap`, and a bucket file written by an + // earlier attempt with a different bucketSpan would land here holding + // keys for a different range. + if (laterPairs[i].key < lo || laterPairs[i].key >= hi) { + Debug(Debug::ERROR) << "Bucket " << b << " of " << tmpPrefix << ".later holds key " + << laterPairs[i].key << ", outside its range [" << lo << ", " + << hi << "). Remove the working directory and re-run the " + << "merge.\n"; + EXIT(EXIT_FAILURE); + } remap[static_cast(laterPairs[i].key - lo)] = laterPairs[i].value; } const std::vector earlierPairs = readBucket(tmpPrefix + ".earlier", b); for (size_t i = 0; i < earlierPairs.size(); i++) { const uint64_t rep = earlierPairs[i].key; + if (rep < lo || rep >= hi) { + Debug(Debug::ERROR) << "Bucket " << b << " of " << tmpPrefix + << ".earlier holds key " << rep << ", outside its range [" + << lo << ", " << hi << "). Remove the working directory and " + << "re-run the merge.\n"; + EXIT(EXIT_FAILURE); + } const uint64_t mapped = remap[static_cast(rep - lo)]; appendPair(buffer, mapped == INVALID ? rep : mapped, earlierPairs[i].value); if (buffer.size() > 32 * 1024 * 1024) { @@ -269,6 +302,12 @@ int mergeclusterparallel(int argc, const char **argv, const Command &command) { par.printParameters(command.cmd, argc, argv, *command.params); const DenseIndex::Info info = DenseIndex::readInfo(seqDb); + // Refused rather than divided by: bucketSpan would be 0 for an empty database + // and `key / bucketSpan` is the first thing partition() does with every row. + if (info.entryCount == 0) { + Debug(Debug::ERROR) << "Database " << seqDb << " is empty\n"; + EXIT(EXIT_FAILURE); + } // One bucket's remap array is 8 bytes per key in its range; size the buckets so // that stays a modest slice of the memory limit. diff --git a/src/linclust/translatecluster.cpp b/src/linclust/translatecluster.cpp index 6b81e987d..c226dc934 100644 --- a/src/linclust/translatecluster.cpp +++ b/src/linclust/translatecluster.cpp @@ -85,7 +85,18 @@ class Buckets { const std::string p = path(prefix, b); if (FileUtil::fileExists(p.c_str()) == false) return out; const size_t bytes = FileUtil::getFileSize(p); - if (bytes == 0 || bytes % sizeof(Pair) != 0) return out; + // An empty bucket is legitimate -- a key range no assignment fell into. + // A *torn* one is not, and returning it as empty silently dropped every + // assignment in that key range, leaving a smaller clustering that still + // looks valid. mergeclusterparallel already makes the identical situation + // fatal, with a comment saying exactly why; the two now agree. + if (bytes % sizeof(Pair) != 0) { + Debug(Debug::ERROR) << "Spill bucket " << p << " is " << bytes << " bytes, not a whole " + << "number of " << sizeof(Pair) << "-byte pairs. It was left by an " + << "interrupted run; remove the spill files and re-run the stage.\n"; + EXIT(EXIT_FAILURE); + } + if (bytes == 0) return out; out.resize(bytes / sizeof(Pair)); FILE *f = FileUtil::openFileOrDie(p.c_str(), "rb", true); if (fread(out.data(), sizeof(Pair), out.size(), f) != out.size()) { @@ -177,6 +188,13 @@ int translatecluster(int argc, const char **argv, const Command &command) { EXIT(EXIT_FAILURE); } const uint64_t subCount = mapBytes / sizeof(uint64_t); + // Refused rather than divided by. span would be 0 for an empty map, and the + // `key / span` that buckets every row is the first thing this does. + if (subCount == 0) { + Debug(Debug::ERROR) << "Key map " << mapFile << " is empty, so there is nothing to " + << "translate into. createrepdb did not finish.\n"; + EXIT(EXIT_FAILURE); + } const uint64_t targetBytes = std::max(Util::computeMemory(par.splitMemoryLimit) / 8, 1ULL * 1024 * 1024); @@ -208,6 +226,18 @@ int translatecluster(int argc, const char **argv, const Command &command) { if (tab == NULL) continue; const uint64_t subRep = strtoull(line, NULL, 10); const uint64_t subMember = strtoull(tab + 1, NULL, 10); + // Validated before it is divided into a bucket index, not after. + // Both columns index `buffers[b]` immediately, so an out-of-range key + // -- a clustering paired with the wrong key map -- wrote past the end + // of the bucket vector before the range check further down could + // report it. + if (subRep >= subCount || subMember >= subCount) { + Debug(Debug::ERROR) << "Clustering " << inTsv << " names sub-key " + << std::max(subRep, subMember) << ", beyond the " << subCount + << " keys in " << mapFile << ". The clustering and the key map " + << "are from different runs.\n"; + EXIT(EXIT_FAILURE); + } byMember.append(static_cast(subMember / span), subMember, subRep); } free(line); @@ -228,7 +258,16 @@ int translatecluster(int argc, const char **argv, const Command &command) { EXIT(EXIT_FAILURE); } const uint64_t origMember = slice[static_cast(pairs[i].key - lo)]; - // Re-bucket on the representative sub-key for the second pass. + // Checked again here: `other` is the column this pass carries + // through untouched, so it was validated on the way in but has + // been through a spill file since, and it is about to index + // byRep's bucket vector. + if (pairs[i].other >= subCount) { + Debug(Debug::ERROR) << "Spill bucket " << b << " holds sub-key " + << pairs[i].other << ", beyond the " << subCount + << " keys in " << mapFile << "\n"; + EXIT(EXIT_FAILURE); + } byRep.append(static_cast(pairs[i].other / span), pairs[i].other, origMember); } diff --git a/src/linclust/translatekeys.cpp b/src/linclust/translatekeys.cpp index b0342ea81..b9dfc8063 100644 --- a/src/linclust/translatekeys.cpp +++ b/src/linclust/translatekeys.cpp @@ -81,6 +81,47 @@ struct TranslatedRow { // schedule(dynamic), so a different schedule -- or a different --threads -- leaves // shards from the earlier attempt on disk, and pass 3 reads every shard it finds. // Both attempts' rows would then be emitted. +// +// Matched against the exact generated grammar, not merely "starts with the +// prefix". The spill prefix is derived from the *output* path, so with +// clusters.tsv.tmp as the output the prefix is clusters.tsv.tmp.bymember -- and a +// plain prefix compare also removed an unrelated neighbouring file called +// clusters.tsv.tmp.bymember-notes. The three shapes actually generated are +// +// . pass 1's fixed-width spill +// .t. pass 2's thread-private spill +// pass 3's output pieces, whose prefix ends in "p" +// +// so the suffix is a run of segments, each `` or `t`, separated by +// dots, with the leading dot optional. Requiring that shape makes the deletion +// exactly as wide as what was created. +bool isGeneratedSpillName(const std::string &name, const std::string &base) { + if (name.size() <= base.size() || name.compare(0, base.size(), base) != 0) { + return false; + } + size_t at = base.size(); + bool first = true; + while (at < name.size()) { + if (name[at] == '.') { + at++; + } else if (first == false) { + return false; // segments after the first must be dot-separated + } + first = false; + if (at < name.size() && name[at] == 't') { + at++; + } + const size_t digitsFrom = at; + while (at < name.size() && name[at] >= '0' && name[at] <= '9') { + at++; + } + if (at == digitsFrom) { + return false; + } + } + return true; +} + void removeSpillFiles(const std::string &prefix) { const size_t slash = prefix.find_last_of('/'); const std::string dir = slash == std::string::npos ? std::string(".") : prefix.substr(0, slash); @@ -90,13 +131,22 @@ void removeSpillFiles(const std::string &prefix) { return; // nothing written yet } struct dirent *entry; + errno = 0; while ((entry = readdir(handle)) != NULL) { const std::string name = entry->d_name; - if (name.size() > base.size() && name.compare(0, base.size(), base) == 0) { + if (isGeneratedSpillName(name, base)) { FileUtil::remove((dir + "/" + name).c_str()); } } - closedir(handle); + const int readErr = errno; + if (readErr != 0) { + Debug(Debug::ERROR) << "Cannot read " << dir << ": " << strerror(readErr) << "\n"; + EXIT(EXIT_FAILURE); + } + if (closedir(handle) != 0) { + Debug(Debug::ERROR) << "Cannot close " << dir << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } } @@ -411,7 +461,6 @@ int translatekeys(int argc, const char **argv, const Command &command) { // are used. uint64_t keyCount = 0; { - LookupCursor probe(lookupFile); // Counting lines is one extra sequential pass, but the alternative is // guessing the key space from the clustering, which need not mention the // highest key at all. @@ -431,22 +480,51 @@ int translatekeys(int argc, const char **argv, const Command &command) { const uint64_t bytesPerKey = 192; const uint64_t perThreadBudget = std::max(targetBytes / static_cast(threads), 1ULL * 1024 * 1024); + // Sized against the *rows* as well as the keys. + // + // A bucket is a range of representative key, but what pass 3 holds is one + // TranslatedRow per *member* assigned into that range -- and cluster sizes are + // skewed, so a bucket of `span` keys can hold far more than `span` rows. Sizing + // on keys alone therefore bounded the accession slice and left the row vector + // unbounded, which is the one that carries a std::string each. + // + // The row count is bounded from the input file: a row is "0\t0\n" at the very + // least. ~128 B per row is the pair of keys plus a short accession and the + // std::string that holds it; being wrong only changes how many buckets are used. + const uint64_t rowsUpperBound = FileUtil::getFileSize(inTsv) / 4 + 1; + const uint64_t bytesPerRow = 128; unsigned int buckets = 1; - while (buckets < 65536 && (keyCount / buckets) * bytesPerKey > perThreadBudget) { + while (buckets < 65536 + && ((keyCount / buckets) * bytesPerKey > perThreadBudget + || (rowsUpperBound / buckets) * bytesPerRow > perThreadBudget)) { buckets *= 2; } const uint64_t span = (keyCount + buckets - 1) / buckets; Debug(Debug::INFO) << "Translating " << keyCount << " keys over " << buckets << " buckets of " << span << "\n"; - const std::string tmpA = outTsv + ".bymember"; - const std::string tmpB = outTsv + ".byrep"; + // Spill files go where --spill-prefix says, which the workflow points inside + // its tmp directory. Deriving them from the output path -- which is what + // happens when the option is not given -- puts every intermediate this command + // writes on the *output* filesystem: at 1e11 the first fixed-width spill alone + // is on the order of 1.6 TB, none of it counted by --scratch-budget, and a + // small output filesystem fills even when scratch has room. + const std::string spillBase = par.spillPrefix.empty() ? outTsv : par.spillPrefix; + const std::string tmpA = spillBase + ".bymember"; + const std::string tmpB = spillBase + ".byrep"; + const std::string piecePrefix = spillBase + ".p"; // Clear anything an interrupted earlier attempt left behind. A rerun only // truncates the shards it happens to touch, and pass 2's thread/bucket // assignment is dynamic, so shards from the previous attempt would otherwise // survive and pass 3 would emit both attempts' rows. removeSpillFiles(tmpA); removeSpillFiles(tmpB); + // The output pieces too. Pass 3 writes .p and then concatenates + // every one it finds, but it `continue`s past a bucket whose key range the + // lookup never covers -- skipping the overwrite, not the concatenation. A + // .p left by an interrupted earlier attempt was therefore appended to + // the final result of the next one. + removeSpillFiles(piecePrefix); // Pass 1: bucket by member key. { @@ -568,10 +646,11 @@ int translatekeys(int argc, const char **argv, const Command &command) { buffer.push_back('\n'); written++; } - FILE *piece = FileUtil::openAndDelete((outTsv + ".p" + SSTR(b)).c_str(), "w"); - writeAllOrDie(buffer.data(), buffer.size(), piece, outTsv); + const std::string piecePath = piecePrefix + SSTR(b); + FILE *piece = FileUtil::openAndDelete(piecePath.c_str(), "w"); + writeAllOrDie(buffer.data(), buffer.size(), piece, piecePath); if (fclose(piece) != 0) { - Debug(Debug::ERROR) << "Cannot close " << outTsv << ".p" << b << "\n"; + Debug(Debug::ERROR) << "Cannot close " << piecePath << "\n"; EXIT(EXIT_FAILURE); } } @@ -579,7 +658,7 @@ int translatekeys(int argc, const char **argv, const Command &command) { FILE *out = FileUtil::openAndDelete(outTsv.c_str(), "w"); std::vector copy(8 << 20); for (unsigned int b = 0; b < buckets; b++) { - const std::string piecePath = outTsv + ".p" + SSTR(b); + const std::string piecePath = piecePrefix + SSTR(b); if (FileUtil::fileExists(piecePath.c_str()) == false) { continue; } diff --git a/src/test/TestParallelCoordination.cpp b/src/test/TestParallelCoordination.cpp index 729b18ff2..c8daa39a5 100644 --- a/src/test/TestParallelCoordination.cpp +++ b/src/test/TestParallelCoordination.cpp @@ -310,6 +310,111 @@ static void testCompletedWorkersNamesOneProducer(const std::string &dir) { check(workers[2] == -1, "an unfinished item names nobody"); } +// Two *independently constructed* objects over the same path, used concurrently +// in one process. +// +// The existing concurrency tests share one object between threads, which the +// object's own mutex serialises. This is the case that mutex could not cover: +// fcntl record locks belong to the process, so two FileLocks for one path both +// believe they hold it, and their separate mutexes serialise nothing. Worse, +// closing *any* descriptor to a file drops the process's locks on it, so one +// object going out of scope silently unlocked the other mid-update. +// +// Without a per-path registry this loses increments; with one the total is exact. +static void testSeparateObjectsOverOnePath(const std::string &dir) { + const std::string path = dir + "/shared_path_counter"; + // Kept to the same scale as the other counter tests: every fetchAdd fsyncs, + // and on a busy journalling filesystem that is milliseconds each. + const int threadCount = 8; + const int perThread = 25; + + std::vector threads; + for (int t = 0; t < threadCount; t++) { + threads.push_back(std::thread([&path, perThread]() { + // A fresh object per thread, and a second short-lived one inside the + // loop whose destruction used to drop the first one's lock. + SharedCounter mine(path); + for (int i = 0; i < perThread; i++) { + mine.fetchAdd(1); + SharedCounter transient(path); + transient.get(); + } + })); + } + for (size_t i = 0; i < threads.size(); i++) { + threads[i].join(); + } + + SharedCounter observer(path); + check(observer.get() == static_cast(threadCount) * perThread, + "separate counter objects over one path do not lose increments"); + + // The same for a queue: two objects, one path, every item claimed once. + const std::string queuePath = dir + "/shared_path_queue"; + const int64_t itemCount = 64; + std::vector claimedBy(static_cast(itemCount), 0); + std::vector > taken(2); + std::vector queueThreads; + for (int t = 0; t < 2; t++) { + queueThreads.push_back(std::thread([&queuePath, itemCount, t, &taken]() { + WorkQueue queue(queuePath, itemCount); + while (true) { + const int64_t item = queue.claim(t + 1, 600); + if (item < 0) { + break; + } + taken[t].push_back(item); + queue.complete(item, t + 1); + } + })); + } + for (size_t i = 0; i < queueThreads.size(); i++) { + queueThreads[i].join(); + } + size_t total = 0; + for (size_t t = 0; t < taken.size(); t++) { + for (size_t i = 0; i < taken[t].size(); i++) { + claimedBy[static_cast(taken[t][i])]++; + } + total += taken[t].size(); + } + bool exactlyOnce = total == static_cast(itemCount); + for (size_t i = 0; i < claimedBy.size(); i++) { + exactlyOnce = exactlyOnce && claimedBy[i] == 1; + } + check(exactlyOnce, "separate queue objects over one path claim every item exactly once"); +} + +// complete() must not mark an item done that another worker now holds. +// +// renew() and release() have always checked ownership; complete() did not, so a +// worker whose lease lapsed -- and whose item another worker is running right now +// -- still recorded it DONE on finishing. The stage then treats an in-progress +// item as complete, and for the reduce it also makes the wrong worker the +// authority for that partition's edge blocks. +static void testCompleteChecksOwnership(const std::string &dir) { + const std::string queuePath = dir + "/ownership_queue"; + WorkQueue queue(queuePath, 2); + + check(queue.claim(1, 1) == 0, "worker 1 takes item 0 under a one-second lease"); + sleep(2); + check(queue.claim(2, 600) == 0, "worker 2 re-claims it once the lease lapses"); + + // Worker 1 finishes late. Its output is superseded by worker 2's, which is + // still running, so this must not be recorded. + queue.complete(0, 1); + check(queue.getDoneCount() == 0, "a lapsed holder cannot complete an item someone else holds"); + + std::vector workers; + check(WorkQueue::readCompletedWorkers(queuePath, workers), "the queue reads back"); + check(workers[0] == -1, "the item is still unfinished"); + + queue.complete(0, 2); + check(queue.getDoneCount() == 1, "the current holder can complete it"); + check(WorkQueue::readCompletedWorkers(queuePath, workers) && workers[0] == 2, + "the current holder is the authority for the item"); +} + int main(int, const char**) { std::string dir = makeTempDir(); @@ -321,6 +426,8 @@ int main(int, const char**) { testReleaseRequeues(dir); testResumeKeepsProgress(dir); testCompletedWorkersNamesOneProducer(dir); + testSeparateObjectsOverOnePath(dir); + testCompleteChecksOwnership(dir); removeTempDir(dir); diff --git a/src/util/createdbparallel.cpp b/src/util/createdbparallel.cpp index 62e030a5e..68f54fefb 100644 --- a/src/util/createdbparallel.cpp +++ b/src/util/createdbparallel.cpp @@ -44,6 +44,7 @@ #include #include +#include #include #ifdef OPENMP @@ -59,12 +60,39 @@ struct Chunk { size_t end; }; +// Per-chunk coordination files live in a subdirectory of 1000, not all in one. +// +// At 1e12 sequences the input is ~350 TB, which is 1.37e6 chunks at the 256 MB +// default and therefore 2.74e6 entries -- a histogram and a plan each -- in a +// single directory. That is a directory no shared filesystem enjoys creating, +// listing or removing. Grouping them keeps any one directory at a thousand files +// while the path stays a pure function of the chunk index, which is what every +// worker relies on. +const size_t COORD_FILES_PER_DIR = 1000; + +std::string chunkGroupDir(const std::string &coordDir, size_t chunkIdx) { + return coordDir + "/c" + SSTR(chunkIdx / COORD_FILES_PER_DIR); +} + +// Racing workers may both create it; only a failure that also leaves no directory +// behind is real. +void ensureChunkGroupDir(const std::string &coordDir, size_t chunkIdx) { + const std::string path = chunkGroupDir(coordDir, chunkIdx); + if (FileUtil::directoryExists(path.c_str()) == true) { + return; + } + if (mkdir(path.c_str(), 0777) != 0 && FileUtil::directoryExists(path.c_str()) == false) { + Debug(Debug::ERROR) << "Cannot create " << path << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } +} + std::string chunkHistPath(const std::string &coordDir, size_t chunkIdx) { - return coordDir + "/chunk." + SSTR(chunkIdx) + ".hist"; + return chunkGroupDir(coordDir, chunkIdx) + "/chunk." + SSTR(chunkIdx) + ".hist"; } std::string chunkPlanPath(const std::string &coordDir, size_t chunkIdx) { - return coordDir + "/chunk." + SSTR(chunkIdx) + ".plan"; + return chunkGroupDir(coordDir, chunkIdx) + "/chunk." + SSTR(chunkIdx) + ".plan"; } void writeAt(int fd, const void *data, size_t length, size_t offset, const char *what) { @@ -134,38 +162,106 @@ size_t findRecordStart(int fd, size_t from, size_t fileSize) { return fileSize; } -// Splits every input file into chunk-size pieces aligned to record boundaries. -// Computed identically and independently by every worker, so the chunk numbering -// -- and therefore the key assignment that breaks length ties by chunk index -- -// never depends on who is running. -std::vector planChunks(const std::vector &filenames, size_t chunkSize) { - std::vector chunks; +// How many chunks each input file is cut into, and where each file's chunks start +// in the global numbering. No I/O at all: a chunk is the piece of the fixed +// `chunkSize` grid it sits on, so the count follows from the file size. +// +// This replaces a planner that resolved *every* chunk boundary up front, in every +// worker, with a 64 KB pread each. At 1e12 (350 TB at the 256 MB default) that is +// 1.37e6 preads -- ~90 GB per worker and ~730 TB across 8192 of them -- paid +// before any worker starts work, every time the stage is entered. +// +// The numbering is still a pure function of (input, chunkSize) and still runs in +// file order, which is what the key assignment's tie-break depends on: ties go to +// the lower chunk index and then to position within the chunk, i.e. input order. +// The grid can leave a chunk empty where one record spans a whole boundary +// region; an empty chunk contributes nothing to the ordering, so the keys are +// unchanged. +struct ChunkLayout { + std::vector countPerFile; + std::vector firstChunkOfFile; + size_t total; + + ChunkLayout() : total(0) {} +}; + +ChunkLayout planChunkLayout(const std::vector &filenames, size_t chunkSize) { + ChunkLayout layout; + layout.countPerFile.assign(filenames.size(), 0); + layout.firstChunkOfFile.assign(filenames.size(), 0); for (size_t fileIdx = 0; fileIdx < filenames.size(); fileIdx++) { const size_t fileSize = FileUtil::getFileSize(filenames[fileIdx]); - if (fileSize == 0) { - continue; - } - const int fd = open(filenames[fileIdx].c_str(), O_RDONLY); - if (fd < 0) { - Debug(Debug::ERROR) << "Cannot open " << filenames[fileIdx] << ": " << strerror(errno) << "\n"; - EXIT(EXIT_FAILURE); - } - size_t begin = 0; - while (begin < fileSize) { - const size_t nominalEnd = std::min(begin + chunkSize, fileSize); - const size_t end = findRecordStart(fd, nominalEnd, fileSize); - if (end > begin) { - Chunk chunk = {fileIdx, begin, end}; - chunks.push_back(chunk); - } - if (end <= begin) { - break; - } - begin = end; + layout.firstChunkOfFile[fileIdx] = layout.total; + layout.countPerFile[fileIdx] = (fileSize + chunkSize - 1) / chunkSize; + layout.total += layout.countPerFile[fileIdx]; + } + return layout; +} + +size_t fileOfChunk(const ChunkLayout &layout, size_t chunkIdx) { + // Few input files in practice, and this runs once per work item. + for (size_t fileIdx = layout.countPerFile.size(); fileIdx > 0; fileIdx--) { + if (chunkIdx >= layout.firstChunkOfFile[fileIdx - 1] + && layout.countPerFile[fileIdx - 1] > 0) { + return fileIdx - 1; } - close(fd); } - return chunks; + return 0; +} + +// Resolves one chunk's byte range: two preads, in the worker that owns it. +Chunk resolveChunk(const std::vector &filenames, const ChunkLayout &layout, + size_t chunkSize, size_t chunkIdx) { + const size_t fileIdx = fileOfChunk(layout, chunkIdx); + const size_t local = chunkIdx - layout.firstChunkOfFile[fileIdx]; + const size_t fileSize = FileUtil::getFileSize(filenames[fileIdx]); + const int fd = open(filenames[fileIdx].c_str(), O_RDONLY); + if (fd < 0) { + Debug(Debug::ERROR) << "Cannot open " << filenames[fileIdx] << ": " << strerror(errno) + << "\n"; + EXIT(EXIT_FAILURE); + } + Chunk chunk; + chunk.fileIdx = fileIdx; + chunk.begin = findRecordStart(fd, local * chunkSize, fileSize); + chunk.end = findRecordStart(fd, std::min((local + 1) * chunkSize, fileSize), fileSize); + close(fd); + if (chunk.end < chunk.begin) { + chunk.end = chunk.begin; + } + return chunk; +} + +// Keeps the chunk count within reach of the coordination machinery. +// +// The default 256 MB is right for the sizes this is usually run at, but at 1e12 +// (350 TB) it derives 1.37e6 chunks -- two work queues of that size, 2.74e6 +// coordination files, and a planner sweep that holds every histogram and plan +// resident (~550 GB) while it runs single-threaded on one node. Scaling the chunk +// size so the count lands near TARGET_CHUNKS makes most of that go away for free +// and costs only a larger per-item buffer, which is bounded by --chunk-size times +// the thread count either way. +// +// Only ever *raises* it: a user who asked for a specific --chunk-size gets it, and +// small inputs keep small chunks so a second worker still has something to claim. +const size_t TARGET_CHUNKS = 100000; + +size_t deriveChunkSize(const std::vector &filenames, size_t requested, + bool userSupplied) { + if (userSupplied) { + return requested; + } + uint64_t totalBytes = 0; + for (size_t i = 0; i < filenames.size(); i++) { + totalBytes += FileUtil::getFileSize(filenames[i]); + } + const uint64_t wanted = totalBytes / TARGET_CHUNKS; + if (wanted <= requested) { + return requested; + } + // Rounded up to a whole number of the requested unit, so the derived value + // stays a recognisable multiple of the documented default. + return static_cast((wanted + requested - 1) / requested) * requested; } // Reads a chunk's bytes so they can be handed to the FASTA parser in one piece. @@ -510,26 +606,41 @@ int createdbparallel(int argc, const char **argv, const Command &command) { FileUtil::makeDir(coordDir.c_str()); } - const std::vector chunks = planChunks(filenames, par.chunkSize); - if (chunks.empty()) { + // Derived once and recorded, so every worker of the run -- including one that + // joins after a restart -- cuts the input on the same grid. A different chunk + // size is a different chunk numbering, and the numbering is what breaks length + // ties when keys are assigned. + const size_t chunkSize = + deriveChunkSize(filenames, par.chunkSize, par.PARAM_CHUNK_SIZE.wasSet); + const ChunkLayout layout = planChunkLayout(filenames, chunkSize); + if (layout.total == 0) { Debug(Debug::ERROR) << "The input files have no entry\n"; EXIT(EXIT_FAILURE); } + if (chunkSize != par.chunkSize) { + Debug(Debug::INFO) << "Using a chunk size of " << chunkSize << " B so the input is " + << layout.total << " chunks rather than " + << "one work item per " << par.chunkSize << " B\n"; + } SharedCounter workerCounter(coordDir + "/worker.counter"); const int64_t workerId = workerCounter.fetchAdd(); - Debug(Debug::INFO) << "Worker " << workerId << " joined, " << chunks.size() << " chunks\n"; + Debug(Debug::INFO) << "Worker " << workerId << " joined, " << layout.total << " chunks\n"; // Pass 1: histogram every chunk. { - WorkQueue scanQueue(coordDir + "/scan.queue", static_cast(chunks.size())); + WorkQueue scanQueue(coordDir + "/scan.queue", static_cast(layout.total)); runQueue(scanQueue, par.threads, workerId, [&](size_t chunkIdx) { const std::string path = chunkHistPath(coordDir, chunkIdx); if (FileUtil::fileExists(path.c_str()) == true) { return; } - ChunkHistogram histogram = scanChunk(filenames[chunks[chunkIdx].fileIdx], - chunks[chunkIdx], chunkIdx); + // Boundaries resolved here, by the chunk's own owner, rather than for + // every chunk in every worker before the stage starts. + const Chunk chunk = resolveChunk(filenames, layout, chunkSize, chunkIdx); + ChunkHistogram histogram = scanChunk(filenames[chunk.fileIdx], chunk, chunkIdx); + histogram.fileIdx = chunk.fileIdx; + ensureChunkGroupDir(coordDir, chunkIdx); histogram.write(path); }); @@ -544,8 +655,8 @@ int createdbparallel(int argc, const char **argv, const Command &command) { planLock.lock(); if (FileUtil::fileExists(planDone.c_str()) == false) { std::vector histograms; - histograms.reserve(chunks.size()); - for (size_t i = 0; i < chunks.size(); i++) { + histograms.reserve(layout.total); + for (size_t i = 0; i < layout.total; i++) { histograms.push_back(ChunkHistogram::read(chunkHistPath(coordDir, i))); } @@ -576,6 +687,7 @@ int createdbparallel(int argc, const char **argv, const Command &command) { EXIT(EXIT_FAILURE); } for (size_t i = 0; i < plans.size(); i++) { + ensureChunkGroupDir(coordDir, plans[i].chunkIdx); plans[i].write(chunkPlanPath(coordDir, plans[i].chunkIdx)); } @@ -629,27 +741,40 @@ int createdbparallel(int argc, const char **argv, const Command &command) { // database, which is addressed by the same dense keys. const int lookupFd = par.writeLookup ? openForWrite(lookupFile) : -1; - WorkQueue emitQueue(coordDir + "/emit.queue", static_cast(chunks.size())); + WorkQueue emitQueue(coordDir + "/emit.queue", static_cast(layout.total)); runQueue(emitQueue, par.threads, workerId, [&](size_t chunkIdx) { const ChunkPlan plan = ChunkPlan::read(chunkPlanPath(coordDir, chunkIdx)); - emitChunk(filenames[chunks[chunkIdx].fileIdx], chunks[chunkIdx], plan, + const Chunk chunk = resolveChunk(filenames, layout, chunkSize, chunkIdx); + emitChunk(filenames[chunk.fileIdx], chunk, plan, seqFd, hdrFd, seqIdxFd, hdrIdxFd, lookupFd); }); // fsync before the sentinel, so a worker that finalises after a crash // cannot read a partially flushed database. - fsync(seqFd); - fsync(hdrFd); - fsync(seqIdxFd); - fsync(hdrIdxFd); - if (lookupFd >= 0) { - fsync(lookupFd); - } - close(seqFd); - close(hdrFd); - close(seqIdxFd); - close(hdrIdxFd); - if (lookupFd >= 0) { - close(lookupFd); + // + // Both the fsync and the close are checked. A delayed ENOSPC, an exhausted + // quota or a shared-filesystem EIO surfaces at exactly these two calls and + // nowhere earlier, and ignoring them let this worker go on to write + // finalize.done over a database with holes in it -- which every later run + // then treats as finished. + static const char *names[] = {"sequence data", "header data", "sequence index", + "header index", "lookup"}; + const int fds[] = {seqFd, hdrFd, seqIdxFd, hdrIdxFd, lookupFd}; + for (size_t i = 0; i < sizeof(fds) / sizeof(fds[0]); i++) { + if (fds[i] < 0) { + continue; + } + if (fsync(fds[i]) != 0) { + const int err = errno; + Debug(Debug::ERROR) << "Cannot flush the " << names[i] << " of " << dataFile << ": " + << strerror(err) << "\n"; + EXIT(EXIT_FAILURE); + } + if (close(fds[i]) != 0) { + const int err = errno; + Debug(Debug::ERROR) << "Cannot close the " << names[i] << " of " << dataFile << ": " + << strerror(err) << "\n"; + EXIT(EXIT_FAILURE); + } } } Debug(Debug::INFO) << "Emit pass done\n"; @@ -669,10 +794,28 @@ int createdbparallel(int argc, const char **argv, const Command &command) { } fclose(typeFile); - DBWriter::writeDbtypeFile(dataFile.c_str(), dbType, par.compressed); - DBWriter::writeDbtypeFile(hdrDataFile.c_str(), Parameters::DBTYPE_GENERIC_DB, par.compressed); - DenseIndex::writeTextIndex(dataFile); - DenseIndex::writeTextIndex(hdrDataFile); + // Never compressed. emitChunk pwrites raw bytes at offsets a plan + // derived from uncompressed lengths fixed, so nothing here compresses + // anything; passing par.compressed through only *labelled* those raw + // files as compressed, and readers then tried to decode them. See the + // note beside createdbparallel's parameter list. + DBWriter::writeDbtypeFile(dataFile.c_str(), dbType, false); + DBWriter::writeDbtypeFile(hdrDataFile.c_str(), Parameters::DBTYPE_GENERIC_DB, false); + // Opt-in. No stage of this pipeline reads a text index -- they all + // address entries through the dense .index.bin -- and it exists only so + // stock MMseqs2 tools can open the database. Measured, the snprintf and + // fwrite loop costs 177 ns per line writing to /dev/null, so at 1e12 + // sequences the two databases are 2e12 lines: ~98 h single-threaded on + // one node, holding the finalize lock, plus ~36 TB of a 1 PB scratch + // budget for output nothing in the run consumes. + if (par.writeTextIndex) { + DenseIndex::writeTextIndex(dataFile); + DenseIndex::writeTextIndex(hdrDataFile); + } else { + Debug(Debug::INFO) << "Skipping the stock-compatible text indices " + << "(--write-text-index 0); the dense .index.bin is what the " + << "distributed stages read\n"; + } Debug(Debug::INFO) << "Database type: " << Parameters::getDbTypeName(dbType) << "\n"; FILE *sentinel = FileUtil::openAndDelete(finalizeDone.c_str(), "w"); diff --git a/src/workflow/LinclustParallel.cpp b/src/workflow/LinclustParallel.cpp index 1d83dc0e6..149d1f467 100644 --- a/src/workflow/LinclustParallel.cpp +++ b/src/workflow/LinclustParallel.cpp @@ -1,5 +1,6 @@ #include "ByteParser.h" #include "CommandCaller.h" +#include "KmerPartition.h" #include "Debug.h" #include "FileUtil.h" #include "Parameters.h" @@ -61,6 +62,20 @@ int linclustparallel(int argc, const char **argv, const Command &command) { EXIT(EXIT_FAILURE); } + // Said once, up front, rather than discovered several stages in. + // + // DBKeyType is a uint32_t unless the build sets MMSEQS_INT64_IDS, whose CMake + // default is off. createdbparallel does refuse an input past the ceiling, but + // only after scanning and planning the whole thing -- hours at the scales this + // command exists for. Naming the limit before anything is launched turns that + // into a one-line answer. + if (static_cast(DB_KEY_INVALID) - 1 < KmerRecord::MAX_ID) { + Debug(Debug::WARNING) + << "This build uses 32-bit database keys, so it can cluster at most " + << static_cast(DB_KEY_INVALID) << " sequences. The distributed pipeline is " + << "designed for 1e11-1e12; build with -DMMSEQS_INT64_IDS=1 for those.\n"; + } + std::string tmpDir = par.db3; std::string hash = SSTR(par.hashParameter(command.databases, par.filenames, par.linclustparallelworkflow)); From db1f30e02e68567c6b8609196a3115d8a0c9ef34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Wed, 5 Aug 2026 13:15:08 +0000 Subject: [PATCH 22/27] Add varint codec and length-rank table Foundations for the packed bucket formats. The length-rank table recovers a sequence length from its key, which the length-ranked key assignment already fixes, so the k-mer record no longer has to carry seqLen. --- src/commons/CMakeLists.txt | 3 + src/commons/LengthRankTable.cpp | 212 +++++++++++++++++++++++++++ src/commons/LengthRankTable.h | 94 ++++++++++++ src/commons/LengthRankedPlan.cpp | 16 ++- src/commons/LengthRankedPlan.h | 10 +- src/commons/VarintCodec.h | 134 ++++++++++++++++++ src/test/CMakeLists.txt | 2 + src/test/TestLengthRankTable.cpp | 236 +++++++++++++++++++++++++++++++ src/test/TestVarintCodec.cpp | 221 +++++++++++++++++++++++++++++ src/util/createdbparallel.cpp | 10 +- 10 files changed, 935 insertions(+), 3 deletions(-) create mode 100644 src/commons/LengthRankTable.cpp create mode 100644 src/commons/LengthRankTable.h create mode 100644 src/commons/VarintCodec.h create mode 100644 src/test/TestLengthRankTable.cpp create mode 100644 src/test/TestVarintCodec.cpp diff --git a/src/commons/CMakeLists.txt b/src/commons/CMakeLists.txt index 52c726c3d..581ea1f41 100644 --- a/src/commons/CMakeLists.txt +++ b/src/commons/CMakeLists.txt @@ -13,6 +13,8 @@ set(commons_header_files commons/DBWriter.h commons/DenseIndex.h commons/LengthRankedPlan.h + commons/LengthRankTable.h + commons/VarintCodec.h commons/IntervalArray.h commons/Debug.h commons/Domain.h @@ -62,6 +64,7 @@ set(commons_source_files commons/DBWriter.cpp commons/DenseIndex.cpp commons/LengthRankedPlan.cpp + commons/LengthRankTable.cpp commons/Debug.cpp commons/ExpressionParser.cpp commons/FileUtil.cpp diff --git a/src/commons/LengthRankTable.cpp b/src/commons/LengthRankTable.cpp new file mode 100644 index 000000000..1af34f91f --- /dev/null +++ b/src/commons/LengthRankTable.cpp @@ -0,0 +1,212 @@ +#include "LengthRankTable.h" + +#include "Debug.h" +#include "FileUtil.h" +#include "Util.h" + +#include +#include +#include +#include + +#include + +namespace { + +struct Header { + uint64_t magic; + uint32_t version; + uint32_t reserved; + uint64_t maxLength; + uint64_t entryCount; + uint64_t runCount; +}; + +} // namespace + +// Out-of-class definitions: C++14 has no inline variables, so an in-class +// initialiser alone is not a definition and any use that takes a reference -- +// std::min, a container insert -- fails to link. +const uint64_t LengthRankTable::MAX_SEQUENCE_LENGTH; +const uint64_t LengthRankTable::MAGIC; +const uint32_t LengthRankTable::VERSION; + +std::string LengthRankTable::fileName(const std::string &dbPath) { return dbPath + ".lenrank"; } + +bool LengthRankTable::exists(const std::string &dbPath) { + return FileUtil::fileExists(fileName(dbPath).c_str()); +} + +void LengthRankTable::write(const std::string &dbPath, const std::vector &runs, + uint64_t entryCount) { + std::vector table; + table.reserve(runs.size()); + uint64_t maxLength = 0; + uint64_t seen = 0; + for (size_t i = 0; i < runs.size(); i++) { + // Strictly descending lengths tiling key space with no hole is what makes + // the lookup a plain binary search. Both are guaranteed by the planner; + // checking here turns a planner regression into an immediate failure + // rather than a subtly wrong length two stages later. + if (i > 0 && runs[i].length >= runs[i - 1].length) { + Debug(Debug::ERROR) << "Length runs are not strictly descending at " << i << ": " + << runs[i - 1].length << " then " << runs[i].length << "\n"; + EXIT(EXIT_FAILURE); + } + if (i > 0 && runs[i].firstKey != runs[i - 1].firstKey + runs[i - 1].count) { + Debug(Debug::ERROR) << "Length runs leave a hole in key space at " << i << "\n"; + EXIT(EXIT_FAILURE); + } + if (runs[i].count == 0) { + Debug(Debug::ERROR) << "Length run " << i << " is empty\n"; + EXIT(EXIT_FAILURE); + } + maxLength = std::max(maxLength, runs[i].length); + seen += runs[i].count; + + Entry entry; + entry.firstKey = runs[i].firstKey; + // Widened to 32 bits so the table itself stays usable for databases the + // k-mer stages would refuse; the 65535 cap is enforced where the 16-bit + // fields actually live, not here. + if (runs[i].length > 0xFFFFFFFFULL) { + Debug(Debug::ERROR) << "Sequence length " << runs[i].length + << " exceeds what the length-rank table can record\n"; + EXIT(EXIT_FAILURE); + } + entry.length = static_cast(runs[i].length); + entry.reserved = 0; + table.push_back(entry); + } + if (seen != entryCount) { + Debug(Debug::ERROR) << "Length runs cover " << seen << " sequences but the database has " + << entryCount << "\n"; + EXIT(EXIT_FAILURE); + } + if (runs.empty() == false && runs[0].firstKey != 0) { + Debug(Debug::ERROR) << "Length runs do not start at key 0\n"; + EXIT(EXIT_FAILURE); + } + + Header header; + header.magic = MAGIC; + header.version = VERSION; + header.reserved = 0; + header.maxLength = maxLength; + header.entryCount = entryCount; + header.runCount = table.size(); + + const std::string finalPath = fileName(dbPath); + const std::string tmpPath = finalPath + ".tmp." + SSTR(getpid()); + FILE *file = fopen(tmpPath.c_str(), "wb"); + if (file == NULL) { + Debug(Debug::ERROR) << "Cannot open " << tmpPath << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + bool ok = fwrite(&header, sizeof(Header), 1, file) == 1; + if (ok && table.empty() == false) { + ok = fwrite(table.data(), sizeof(Entry), table.size(), file) == table.size(); + } + if (ok == false || fflush(file) != 0 || fsync(fileno(file)) != 0) { + Debug(Debug::ERROR) << "Cannot write " << tmpPath << ": " << strerror(errno) << "\n"; + fclose(file); + EXIT(EXIT_FAILURE); + } + if (fclose(file) != 0) { + Debug(Debug::ERROR) << "Cannot close " << tmpPath << ": " << strerror(errno) << "\n"; + EXIT(EXIT_FAILURE); + } + FileUtil::move(tmpPath.c_str(), finalPath.c_str()); +} + +void LengthRankTable::open(const std::string &dbPath) { + const std::string path = fileName(dbPath); + FILE *file = fopen(path.c_str(), "rb"); + if (file == NULL) { + Debug(Debug::ERROR) << "Cannot open length-rank table " << path << ": " << strerror(errno) + << "\nIt is written by createdbparallel; a database built before this " + "existed must be rebuilt.\n"; + EXIT(EXIT_FAILURE); + } + Header header; + if (fread(&header, sizeof(Header), 1, file) != 1) { + Debug(Debug::ERROR) << "Length-rank table " << path << " is truncated\n"; + fclose(file); + EXIT(EXIT_FAILURE); + } + if (header.magic != MAGIC || header.version != VERSION) { + Debug(Debug::ERROR) << "Length-rank table " << path << " has a bad magic or version\n"; + fclose(file); + EXIT(EXIT_FAILURE); + } + maxLength = header.maxLength; + entryCount = header.entryCount; + entries.resize(static_cast(header.runCount)); + if (entries.empty() == false && + fread(entries.data(), sizeof(Entry), entries.size(), file) != entries.size()) { + Debug(Debug::ERROR) << "Length-rank table " << path << " is short: expected " + << entries.size() << " runs\n"; + fclose(file); + EXIT(EXIT_FAILURE); + } + // Nothing may follow. A longer file means this is not the format we think it + // is, and reading it as though it were would give plausible wrong lengths. + char extra = 0; + const bool trailing = fread(&extra, 1, 1, file) == 1; + fclose(file); + if (trailing) { + Debug(Debug::ERROR) << "Length-rank table " << path << " has trailing bytes\n"; + EXIT(EXIT_FAILURE); + } + // The invariants the binary search depends on. A table that violates them + // returns plausible wrong lengths rather than failing, so they are checked on + // open rather than assumed -- it costs one pass over at most a few thousand + // entries, once per process. + for (size_t i = 0; i < entries.size(); i++) { + if (i > 0 && (entries[i].firstKey <= entries[i - 1].firstKey || + entries[i].length >= entries[i - 1].length)) { + Debug(Debug::ERROR) << "Length-rank table " << path << " is not ordered at run " << i + << "\n"; + EXIT(EXIT_FAILURE); + } + if (entries[i].firstKey >= entryCount) { + Debug(Debug::ERROR) << "Length-rank table " << path << " run " << i + << " starts past the end of the database\n"; + EXIT(EXIT_FAILURE); + } + } + if (entries.empty() == false && entries[0].firstKey != 0) { + Debug(Debug::ERROR) << "Length-rank table " << path << " does not start at key 0\n"; + EXIT(EXIT_FAILURE); + } +} + +bool LengthRankTable::tryLengthOf(uint64_t key, unsigned int &length) const { + if (entries.empty() || key >= entryCount) { + return false; + } + // Runs are ordered by ascending firstKey, so the run owning a key is the last + // one that starts at or below it. + size_t lo = 0; + size_t hi = entries.size(); + while (lo + 1 < hi) { + const size_t mid = lo + (hi - lo) / 2; + if (entries[mid].firstKey <= key) { + lo = mid; + } else { + hi = mid; + } + } + length = static_cast(entries[lo].length); + return true; +} + +unsigned int LengthRankTable::lengthOf(uint64_t key) const { + unsigned int length = 0; + if (tryLengthOf(key, length) == false) { + Debug(Debug::ERROR) << "Key " << key << " has no length in a table of " << entryCount + << " sequences\n"; + EXIT(EXIT_FAILURE); + } + return length; +} diff --git a/src/commons/LengthRankTable.h b/src/commons/LengthRankTable.h new file mode 100644 index 000000000..3b6a9f881 --- /dev/null +++ b/src/commons/LengthRankTable.h @@ -0,0 +1,94 @@ +#ifndef MMSEQS_LENGTHRANKTABLE_H +#define MMSEQS_LENGTHRANKTABLE_H + +#include +#include +#include + +// Sequence length from a database key, in space proportional to the number of +// distinct lengths rather than to the database. +// +// This exists to delete a field from the k-mer record. Stock kmermatcher keeps +// `seqkey_to_len[dbKeySize]` (kmermatcher.cpp:1236), an array sized by *key +// space* -- 2-4 TB at 1e12 -- so the distributed map carried a 2-byte `seqLen` in +// every k-mer record instead. At 21 k-mers per sequence that is 42 B/seq, or +// 42 TB at 1e12, spent re-stating something the key already determines. +// +// It already determines it because `createdbparallel` assigns keys **longest +// sequence first**: key == global length rank. So all sequences of one length +// occupy a contiguous key range, and the entire key -> length map is one +// (firstKey, length) pair per distinct length, ordered. +// +// Sparse rather than an array indexed by length: the entry count is then bounded +// by the number of distinct lengths, not by the longest sequence. For proteins +// capped at 65535 residues that is at most 1 MB either way, but a dense array +// would scale with a single long contig and this does not. +// +// The pairs come free: `createdbparallel` pass 1 already histograms lengths in +// order to compute byte offsets, and `buildLengthRankedPlan` hands them over as it +// assigns the keys they describe -- so the table cannot drift from the key order +// it encodes. +class LengthRankTable { +public: + // One distinct length and where its keys start, longest first. + struct Run { + uint64_t length; + uint64_t firstKey; + uint64_t count; + }; + + // On disk and in memory: ordered by ascending firstKey, which is the same as + // descending length. + struct Entry { + uint64_t firstKey; + uint32_t length; + uint32_t reserved; + }; + + LengthRankTable() : maxLength(0), entryCount(0) {} + + // .lenrank + static std::string fileName(const std::string &dbPath); + + // Writes to a private sibling and renames, so an interrupted build never + // leaves a short file that a later stage reads as a valid (and wrong) table. + // Runs must be sorted by descending length and must tile key space. + static void write(const std::string &dbPath, const std::vector &runs, uint64_t entryCount); + + static bool exists(const std::string &dbPath); + + // Exits if the file is missing, truncated, or fails its magic/version check. + // A silently wrong length would mis-rank k-mer group centres -- a different + // clustering with no error anywhere, which is the failure this pipeline is + // least able to detect, so it is never tolerated. + void open(const std::string &dbPath); + + bool isOpen() const { return entries.empty() == false; } + + uint64_t getEntryCount() const { return entryCount; } + uint64_t getMaxLength() const { return maxLength; } + size_t getRunCount() const { return entries.size(); } + + // Length of the sequence with this key. Exits if the key is outside the + // database. + unsigned int lengthOf(uint64_t key) const; + + // Non-fatal form, for validating the table against a database rather than + // trusting it. + bool tryLengthOf(uint64_t key, unsigned int &length) const; + + // The 16-bit `pos` and the historic 16-bit `seqLen` of a k-mer record cap + // sequences here. Nothing enforced it before: a longer sequence wrapped + // silently and its k-mers were extracted at wrong positions. + static const uint64_t MAX_SEQUENCE_LENGTH = 65535; + + static const uint64_t MAGIC = 0x4B4E41524E454C4DULL; + static const uint32_t VERSION = 1; + +private: + std::vector entries; + uint64_t maxLength; + uint64_t entryCount; +}; + +#endif diff --git a/src/commons/LengthRankedPlan.cpp b/src/commons/LengthRankedPlan.cpp index 65f5b692e..7b010dacf 100644 --- a/src/commons/LengthRankedPlan.cpp +++ b/src/commons/LengthRankedPlan.cpp @@ -175,7 +175,8 @@ ChunkPlan ChunkPlan::read(const std::string &path) { } LengthRankedTotals buildLengthRankedPlan(std::vector &histograms, - std::vector &plans) { + std::vector &plans, + std::vector *lengthRuns) { // Order by chunk index so the plan never depends on the order the histograms // happened to be collected in. std::sort(histograms.begin(), histograms.end(), @@ -221,6 +222,12 @@ LengthRankedTotals buildLengthRankedPlan(std::vector &histograms for (size_t l = 0; l < lengths.size(); l++) { const uint64_t length = lengths[l]; + // Keys are handed out longest first, so at this point nextKey is exactly + // the number of sequences longer than `length` -- which is the one value + // the length-rank table needs per distinct length. Captured here rather + // than recomputed, so the table cannot drift from the key assignment it + // describes. + const uint64_t runFirstKey = nextKey; for (size_t i = 0; i < histograms.size(); i++) { if (cursor[i] == 0 || histograms[i].buckets[cursor[i] - 1].length != length) { continue; @@ -252,6 +259,13 @@ LengthRankedTotals buildLengthRankedPlan(std::vector &histograms nextDataOffset += bucket.count * (length + 2); nextHdrOffset += bucket.headerBytes; } + if (lengthRuns != NULL && nextKey > runFirstKey) { + LengthRankTable::Run run; + run.length = length; + run.firstKey = runFirstKey; + run.count = nextKey - runFirstKey; + lengthRuns->push_back(run); + } } for (size_t i = 0; i < histograms.size(); i++) { diff --git a/src/commons/LengthRankedPlan.h b/src/commons/LengthRankedPlan.h index 8781e7479..9d3df89bd 100644 --- a/src/commons/LengthRankedPlan.h +++ b/src/commons/LengthRankedPlan.h @@ -5,6 +5,8 @@ #include #include +#include "LengthRankTable.h" + // Placement plan for a length-ranked, densely-keyed sequence database that many // independent workers build into one set of output files without ever merging. // @@ -126,7 +128,13 @@ struct LengthRankedTotals { // Input histograms may arrive in any order; they are ordered by chunkIdx here so // the result does not depend on directory listing order. plans is resized to one // entry per histogram, indexed the same way as the sorted chunk order. +// lengthRuns, when non-NULL, receives one entry per distinct sequence length in +// descending length order: the length, the first key that has it, and how many +// sequences do. That is the whole content of the length-rank table, and it is +// produced here rather than recomputed later because it is a by-product of the +// key assignment itself -- deriving it separately would let the two disagree. LengthRankedTotals buildLengthRankedPlan(std::vector &histograms, - std::vector &plans); + std::vector &plans, + std::vector *lengthRuns = NULL); #endif diff --git a/src/commons/VarintCodec.h b/src/commons/VarintCodec.h new file mode 100644 index 000000000..15c31446b --- /dev/null +++ b/src/commons/VarintCodec.h @@ -0,0 +1,134 @@ +#ifndef MMSEQS_VARINTCODEC_H +#define MMSEQS_VARINTCODEC_H + +#include +#include + +// LEB128 varints, zigzag mapping and narrow fixed-width integers, for the k-mer +// and candidate-edge bucket formats. +// +// Why those intermediates are encoded at all. They are the two largest things +// this pipeline writes, and at the target scales they are what decides whether a +// run fits its scratch filesystem. With fixed-width records, 1e12 sequences give +// ~504 TB of k-mer shuffle (21 x 24 B) and several hundred TB of candidate edges +// (~22 x 17 B measured at 1e9) against a budget of roughly 1 PB. Waves divide the +// shuffle but not the edges, because alignment runs once after every wave. +// +// Both formats are written once and read back sequentially exactly once, which is +// precisely the case where a variable-width encoding costs nothing but a little +// CPU: nothing seeks into them, nothing updates them in place. +// +// LEB128 rather than group- or stream-vbyte: the fields here are heterogeneous (a +// packed k-mer, an ascending id delta, a position, a zigzagged diagonal), so they +// do not form the uniform four-lane groups those layouts need, and the branchy +// decode is far from the bottleneck next to the sort and the alignment it feeds. +namespace VarintCodec { + +// A 64-bit value is at most ten 7-bit groups. +const size_t MAX_BYTES = 10; + +// Bytes write() will emit for this value. Used to size buffers up front, so the +// encoders never have to test for capacity per field. +inline size_t size(uint64_t value) { + size_t n = 1; + while (value >= 0x80) { + value >>= 7; + n++; + } + return n; +} + +// Appends value and returns the new write position. The caller guarantees +// MAX_BYTES of room -- every producer here writes into a buffer sized with size() +// or with a MAX_BYTES-per-field bound. +inline uint8_t *write(uint8_t *out, uint64_t value) { + while (value >= 0x80) { + *out++ = static_cast((value & 0x7F) | 0x80); + value >>= 7; + } + *out++ = static_cast(value); + return out; +} + +// Bounds-checked decode. Returns false on a truncated or over-long encoding and +// leaves `in` untouched, so a torn block tail is a clean failure rather than a +// read past the buffer. +// +// This is not defensive programming for its own sake: a worker killed mid-flush +// leaves exactly such a tail, and the readers are required to round down to whole +// records and carry on rather than treat the partition as poisoned. +inline bool read(const uint8_t *&in, const uint8_t *end, uint64_t &value) { + uint64_t result = 0; + unsigned int shift = 0; + const uint8_t *p = in; + while (p < end) { + const uint8_t byte = *p++; + // The tenth group carries only the single remaining bit; anything wider + // is a corrupt encoding, not a large number. + if (shift == 63 && (byte & 0xFE) != 0) { + return false; + } + result |= static_cast(byte & 0x7F) << shift; + if ((byte & 0x80) == 0) { + value = result; + in = p; + return true; + } + shift += 7; + if (shift > 63) { + return false; + } + } + return false; +} + +// Maps a signed value onto an unsigned one that is small when the input is small +// in magnitude, so a varint stays short for both signs. Diagonals are signed and +// cluster near zero; without this every negative one would encode as ten bytes. +inline uint64_t zigzag(int64_t value) { + return (static_cast(value) << 1) ^ static_cast(-(value < 0 ? 1 : 0)); +} + +inline int64_t unzigzag(uint64_t value) { + return static_cast(value >> 1) ^ -static_cast(value & 1); +} + +// Narrowest byte count that can hold every value up to maxValue. The k-mer field +// uses this: at --min-seq-id 0.9 the index is base-13 over k = 14, so 13^14 is +// 2^51.8 and seven bytes are enough, while --min-seq-id >= 0.99 switches to the +// 21-letter alphabet (2^61.5) and needs eight. +inline unsigned int fixedWidthFor(uint64_t maxValue) { + unsigned int width = 1; + while (width < 8 && (maxValue >> (width * 8)) != 0) { + width++; + } + return width; +} + +// Little-endian fixed width. Used where the value has a known ceiling but no +// useful skew towards small numbers, so a varint would only add a continuation +// bit per byte. +inline uint8_t *writeFixed(uint8_t *out, uint64_t value, unsigned int width) { + for (unsigned int i = 0; i < width; i++) { + *out++ = static_cast(value & 0xFF); + value >>= 8; + } + return out; +} + +inline bool readFixed(const uint8_t *&in, const uint8_t *end, unsigned int width, uint64_t &value) { + if (static_cast(end - in) < width) { + return false; + } + uint64_t result = 0; + for (unsigned int i = 0; i < width; i++) { + result |= static_cast(in[i]) << (i * 8); + } + in += width; + value = result; + return true; +} + +} // namespace VarintCodec + +#endif diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 42e3f1013..9c8f99d2b 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -26,6 +26,8 @@ set(TESTS TestDenseIndex.cpp TestKmerPartition.cpp TestLengthRankedPlan.cpp + TestVarintCodec.cpp + TestLengthRankTable.cpp TestReduceMatrix.cpp TestScoreMatrixSerialization.cpp TestSequenceIndex.cpp diff --git a/src/test/TestLengthRankTable.cpp b/src/test/TestLengthRankTable.cpp new file mode 100644 index 000000000..6af9b9762 --- /dev/null +++ b/src/test/TestLengthRankTable.cpp @@ -0,0 +1,236 @@ +// Tests for the key -> length table that lets the k-mer record drop its seqLen +// field. +// +// The property under test is simple to state and expensive to get wrong: for +// every key in the database, the table must return exactly the length the +// length-ranked key assignment gave that key. A table that is merely *close* does +// not fail visibly -- it mis-ranks the centre of a k-mer group, which produces a +// different clustering with no error message. So the round-trip here is checked +// exhaustively against a brute-force expectation rather than sampled. + +#include "LengthRankTable.h" + +#include +#include +#include +#include +#include + +#include + +#include +#include + +const char* binary_name = "test_lengthranktable"; + +static int failures = 0; + +static void check(bool condition, const std::string &what) { + if (condition == false) { + fprintf(stderr, "FAIL: %s\n", what.c_str()); + failures++; + } else { + fprintf(stdout, "ok: %s\n", what.c_str()); + } +} + +static std::string makeTempDir() { + char tmpl[] = "/tmp/mmseqs_lenrank_testXXXXXX"; + char *dir = mkdtemp(tmpl); + if (dir == NULL) { + perror("mkdtemp"); + exit(EXIT_FAILURE); + } + return std::string(dir); +} + +static uint64_t nextRandom(uint64_t &state) { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + return state; +} + +// Turns a multiset of sequence lengths into the runs the planner would produce: +// keys handed out longest first, so a length's run starts at the number of +// strictly longer sequences. +static std::vector runsFor(std::vector lengths) { + std::sort(lengths.begin(), lengths.end(), std::greater()); + std::vector runs; + size_t i = 0; + while (i < lengths.size()) { + size_t j = i; + while (j < lengths.size() && lengths[j] == lengths[i]) { + j++; + } + LengthRankTable::Run run; + run.length = lengths[i]; + run.firstKey = i; + run.count = j - i; + runs.push_back(run); + i = j; + } + return runs; +} + +// The brute-force expectation: sort descending, and key k has the length at +// position k. This is the definition of a length-ranked key. +static void checkRoundTrip(const std::string &dir, const std::string &name, + std::vector lengths, const std::string &what) { + const std::string db = dir + "/" + name; + std::vector sorted = lengths; + std::sort(sorted.begin(), sorted.end(), std::greater()); + + LengthRankTable::write(db, runsFor(lengths), sorted.size()); + LengthRankTable table; + table.open(db); + + bool ok = table.getEntryCount() == sorted.size(); + ok = ok && table.getMaxLength() == (sorted.empty() ? 0 : sorted[0]); + for (size_t k = 0; k < sorted.size(); k++) { + unsigned int got = 0; + if (table.tryLengthOf(k, got) == false || got != sorted[k]) { + ok = false; + break; + } + } + // One past the end must be refused, not clamped: a key outside the database + // is a caller bug, and returning the last length would hide it. + unsigned int ignored = 0; + ok = ok && table.tryLengthOf(sorted.size(), ignored) == false; + check(ok, what); +} + +static void testRoundTrips(const std::string &dir) { + // One sequence. + checkRoundTrip(dir, "single", std::vector(1, 42), "one sequence round-trips"); + + // Every sequence the same length: one run covering the whole key space. + checkRoundTrip(dir, "uniform", std::vector(1000, 300), + "1000 sequences of one length round-trip"); + + // Every sequence a distinct length: one run per key, the worst case for the + // run count. + std::vector distinct; + for (uint64_t l = 1; l <= 2000; l++) { + distinct.push_back(l); + } + checkRoundTrip(dir, "distinct", distinct, "2000 distinct lengths round-trip"); + + // A realistic protein-ish distribution: heavy repetition, wide gaps, a long + // tail. Gaps matter because a length with no sequences must never be returned. + std::vector realistic; + uint64_t state = 0x243F6A8885A308D3ULL; + for (int i = 0; i < 20000; i++) { + const uint64_t bucket = nextRandom(state) % 100; + if (bucket < 70) { + realistic.push_back(100 + nextRandom(state) % 300); + } else if (bucket < 95) { + realistic.push_back(400 + nextRandom(state) % 2000); + } else { + realistic.push_back(3000 + nextRandom(state) % 60000); + } + } + checkRoundTrip(dir, "realistic", realistic, + "20000 sequences over a skewed length distribution round-trip"); + + // Length 1 present, and a maximum at the 16-bit ceiling the k-mer record + // imposes. + std::vector extremes; + extremes.push_back(1); + extremes.push_back(1); + extremes.push_back(LengthRankTable::MAX_SEQUENCE_LENGTH); + extremes.push_back(2); + checkRoundTrip(dir, "extremes", extremes, "length 1 and the 65535 ceiling round-trip"); +} + +static void testCorruptionRejected(const std::string &dir) { + const std::string db = dir + "/corrupt"; + std::vector lengths; + for (uint64_t l = 1; l <= 50; l++) { + lengths.push_back(l); + lengths.push_back(l); + } + LengthRankTable::write(db, runsFor(lengths), lengths.size()); + const std::string path = LengthRankTable::fileName(db); + + // Baseline: the untouched file opens. + { + LengthRankTable table; + table.open(db); + check(table.getRunCount() == 50, "a table of 50 distinct lengths has 50 runs"); + } + + std::vector good; + { + FILE *f = fopen(path.c_str(), "rb"); + char buffer[4096]; + size_t got = 0; + while ((got = fread(buffer, 1, sizeof(buffer), f)) > 0) { + good.insert(good.end(), buffer, buffer + got); + } + fclose(f); + } + + // Each of these must be rejected. They are checked by running a child + // process, because the reader's contract is to exit rather than return an + // error -- a silently wrong length table is the one outcome that must be + // impossible. + struct Case { + const char *what; + int kind; // 0 truncate, 1 magic, 2 trailing + }; + const Case cases[] = { + {"a truncated length-rank table is rejected", 0}, + {"a bad magic is rejected", 1}, + {"trailing bytes are rejected", 2}, + }; + for (size_t c = 0; c < sizeof(cases) / sizeof(cases[0]); c++) { + std::vector broken = good; + if (cases[c].kind == 0) { + broken.resize(broken.size() - 8); + } else if (cases[c].kind == 1) { + broken[0] = static_cast(broken[0] ^ 0xFF); + } else { + broken.push_back('x'); + } + FILE *f = fopen(path.c_str(), "wb"); + fwrite(broken.data(), 1, broken.size(), f); + fclose(f); + + fflush(stdout); + fflush(stderr); + const pid_t pid = fork(); + if (pid == 0) { + // Silence the expected error message so the log stays readable. + freopen("/dev/null", "w", stderr); + LengthRankTable table; + table.open(db); + _exit(0); // reached only if the corruption was accepted + } + int status = 0; + waitpid(pid, &status, 0); + const bool rejected = !(WIFEXITED(status) && WEXITSTATUS(status) == 0); + check(rejected, cases[c].what); + } +} + +int main(int, const char**) { + const std::string dir = makeTempDir(); + + testRoundTrips(dir); + testCorruptionRejected(dir); + + // Best-effort cleanup; the test owns a fresh mkdtemp directory. + std::string cmd = "rm -rf '" + dir + "'"; + if (system(cmd.c_str()) != 0) { + fprintf(stderr, "warning: could not remove %s\n", dir.c_str()); + } + + if (failures > 0) { + fprintf(stderr, "\n%d check(s) failed\n", failures); + return EXIT_FAILURE; + } + fprintf(stdout, "\nall checks passed\n"); + return EXIT_SUCCESS; +} diff --git a/src/test/TestVarintCodec.cpp b/src/test/TestVarintCodec.cpp new file mode 100644 index 000000000..0a8a2ce97 --- /dev/null +++ b/src/test/TestVarintCodec.cpp @@ -0,0 +1,221 @@ +// Tests for the varint / zigzag / fixed-width codec the k-mer and candidate-edge +// bucket formats are built on. +// +// Two properties matter, and they fail in opposite directions: +// +// - round-trip exactness. These encodings sit under the two largest +// intermediates in the pipeline, and a value that decodes to something else +// does not crash -- it produces a k-mer that groups with the wrong partners, +// or an edge on the wrong diagonal. That is a silently different clustering, +// which is the failure mode this whole project is least able to detect. +// - clean rejection of a truncated or corrupt encoding. A worker killed +// mid-flush leaves a torn tail, and readers are required to round down to +// whole records and carry on. That is only safe if the decoder reliably says +// "no" instead of reading past the buffer. + +#include "VarintCodec.h" + +#include +#include +#include +#include + +const char* binary_name = "test_varintcodec"; + +static int failures = 0; + +static void check(bool condition, const std::string &what) { + if (condition == false) { + fprintf(stderr, "FAIL: %s\n", what.c_str()); + failures++; + } else { + fprintf(stdout, "ok: %s\n", what.c_str()); + } +} + +// A cheap deterministic generator, so a failure is reproducible without carrying +// a seed corpus around. +static uint64_t nextRandom(uint64_t &state) { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + return state; +} + +static void testUnsignedRoundTrip() { + std::vector values; + // Every power-of-two boundary, and both sides of it: the byte count changes + // at 2^7, 2^14, ... and off-by-one there is the classic varint defect. + for (unsigned int bit = 0; bit < 64; bit++) { + const uint64_t v = static_cast(1) << bit; + values.push_back(v - 1); + values.push_back(v); + values.push_back(v + 1); + } + values.push_back(0); + values.push_back(UINT64_MAX); + values.push_back(UINT64_MAX - 1); + uint64_t state = 0x9E3779B97F4A7C15ULL; + for (int i = 0; i < 20000; i++) { + values.push_back(nextRandom(state)); + // Also small values, which the random stream almost never produces and + // which are the common case in every real field. + values.push_back(nextRandom(state) % 256); + } + + bool allOk = true; + bool sizeOk = true; + for (size_t i = 0; i < values.size(); i++) { + uint8_t buffer[VarintCodec::MAX_BYTES]; + uint8_t *end = VarintCodec::write(buffer, values[i]); + const size_t written = static_cast(end - buffer); + if (written != VarintCodec::size(values[i]) || written > VarintCodec::MAX_BYTES) { + sizeOk = false; + } + const uint8_t *in = buffer; + uint64_t decoded = 0; + if (VarintCodec::read(in, end, decoded) == false || decoded != values[i] || + in != end) { + allOk = false; + } + } + check(allOk, "varint round-trips every boundary and 40000 random values"); + check(sizeOk, "size() agrees with write() and never exceeds MAX_BYTES"); +} + +static void testZigzagRoundTrip() { + std::vector values; + values.push_back(0); + values.push_back(1); + values.push_back(-1); + values.push_back(INT64_MAX); + values.push_back(INT64_MIN); + // The diagonal field is an int16_t, so its whole range is worth covering + // exhaustively rather than sampling. + for (int v = -32768; v <= 32767; v++) { + values.push_back(v); + } + + bool allOk = true; + bool smallOk = true; + for (size_t i = 0; i < values.size(); i++) { + const uint64_t mapped = VarintCodec::zigzag(values[i]); + if (VarintCodec::unzigzag(mapped) != values[i]) { + allOk = false; + } + // The point of zigzag: a small negative must stay short. Without it every + // negative diagonal would encode as ten bytes. + if (values[i] >= -63 && values[i] <= 63 && VarintCodec::size(mapped) != 1) { + smallOk = false; + } + } + check(allOk, "zigzag round-trips the full int16 range and the int64 extremes"); + check(smallOk, "zigzag keeps small-magnitude negatives to one varint byte"); +} + +static void testTruncationRejected() { + // A ten-byte value, then every proper prefix of it. Each must be refused + // rather than decoded as some shorter number. + uint8_t buffer[VarintCodec::MAX_BYTES]; + uint8_t *end = VarintCodec::write(buffer, UINT64_MAX); + const size_t full = static_cast(end - buffer); + + bool allRejected = true; + for (size_t cut = 0; cut < full; cut++) { + const uint8_t *in = buffer; + uint64_t decoded = 0; + if (VarintCodec::read(in, buffer + cut, decoded)) { + allRejected = false; + } + // A rejected read must not consume anything, or a caller that retries + // with more data would resume from the wrong offset. + if (in != buffer) { + allRejected = false; + } + } + check(allRejected, "every truncated prefix of a 10-byte varint is rejected, cursor unmoved"); + + // An encoding that continues past 64 bits is corrupt, not a big number. + uint8_t overlong[12]; + for (size_t i = 0; i < 11; i++) { + overlong[i] = 0xFF; + } + overlong[11] = 0x01; + const uint8_t *in = overlong; + uint64_t decoded = 0; + check(VarintCodec::read(in, overlong + 12, decoded) == false, + "an over-long varint is rejected rather than silently truncated"); + + // The tenth group may carry only bit 63. + uint8_t tenth[VarintCodec::MAX_BYTES]; + for (size_t i = 0; i < 9; i++) { + tenth[i] = 0x80; + } + tenth[9] = 0x02; // bit 1 of the tenth group is past bit 64 + const uint8_t *in2 = tenth; + check(VarintCodec::read(in2, tenth + VarintCodec::MAX_BYTES, decoded) == false, + "a tenth group carrying more than one bit is rejected"); +} + +static void testFixedWidth() { + bool widthOk = true; + widthOk = widthOk && VarintCodec::fixedWidthFor(0) == 1; + widthOk = widthOk && VarintCodec::fixedWidthFor(255) == 1; + widthOk = widthOk && VarintCodec::fixedWidthFor(256) == 2; + // The two cases the k-mer field actually takes: 13^14 at --min-seq-id 0.9 and + // 21^14 at >= 0.99. Seven bytes and eight respectively -- the whole reason the + // width is derived rather than fixed at 8. + uint64_t base13 = 1; + uint64_t base21 = 1; + for (int i = 0; i < 14; i++) { + base13 *= 13; + base21 *= 21; + } + widthOk = widthOk && VarintCodec::fixedWidthFor(base13 - 1) == 7; + widthOk = widthOk && VarintCodec::fixedWidthFor(base21 - 1) == 8; + check(widthOk, "fixedWidthFor picks 7 bytes for a base-13 k=14 k-mer and 8 for base-21"); + + bool roundTripOk = true; + bool truncationOk = true; + uint64_t state = 0xDEADBEEFCAFEBABEULL; + for (unsigned int width = 1; width <= 8; width++) { + const uint64_t mask = + (width == 8) ? UINT64_MAX : ((static_cast(1) << (width * 8)) - 1); + for (int i = 0; i < 2000; i++) { + const uint64_t value = nextRandom(state) & mask; + uint8_t buffer[8]; + uint8_t *end = VarintCodec::writeFixed(buffer, value, width); + if (static_cast(end - buffer) != width) { + roundTripOk = false; + } + const uint8_t *in = buffer; + uint64_t decoded = 0; + if (VarintCodec::readFixed(in, end, width, decoded) == false || decoded != value || + in != end) { + roundTripOk = false; + } + // One byte short must be refused. + const uint8_t *shortIn = buffer; + uint64_t ignored = 0; + if (VarintCodec::readFixed(shortIn, buffer + width - 1, width, ignored)) { + truncationOk = false; + } + } + } + check(roundTripOk, "fixed-width round-trips every width from 1 to 8 bytes"); + check(truncationOk, "a short fixed-width field is rejected"); +} + +int main(int, const char**) { + testUnsignedRoundTrip(); + testZigzagRoundTrip(); + testTruncationRejected(); + testFixedWidth(); + + if (failures > 0) { + fprintf(stderr, "\n%d check(s) failed\n", failures); + return EXIT_FAILURE; + } + fprintf(stdout, "\nall checks passed\n"); + return EXIT_SUCCESS; +} diff --git a/src/util/createdbparallel.cpp b/src/util/createdbparallel.cpp index 68f54fefb..b353a6b36 100644 --- a/src/util/createdbparallel.cpp +++ b/src/util/createdbparallel.cpp @@ -32,6 +32,7 @@ #include "FileUtil.h" #include "KSeqWrapper.h" #include "KmerPartition.h" +#include "LengthRankTable.h" #include "LengthRankedPlan.h" #include "ParallelCoordination.h" #include "Parameters.h" @@ -661,7 +662,8 @@ int createdbparallel(int argc, const char **argv, const Command &command) { } std::vector plans; - const LengthRankedTotals totals = buildLengthRankedPlan(histograms, plans); + std::vector lengthRuns; + const LengthRankedTotals totals = buildLengthRankedPlan(histograms, plans, &lengthRuns); if (totals.seqCount == 0) { Debug(Debug::ERROR) << "The input files have no entry\n"; EXIT(EXIT_FAILURE); @@ -691,6 +693,12 @@ int createdbparallel(int argc, const char **argv, const Command &command) { plans[i].write(chunkPlanPath(coordDir, plans[i].chunkIdx)); } + // The key -> length table. It is a by-product of the key assignment + // just computed, and it is what lets the k-mer record drop its 2-byte + // seqLen field -- 42 B per sequence, 42 TB at 1e12 -- without + // reintroducing stock's key-space-sized seqkey_to_len array. + LengthRankTable::write(dataFile, lengthRuns, totals.seqCount); + allocateFile(dataFile, totals.dataBytes); allocateFile(hdrDataFile, totals.headerBytes); if (par.writeLookup) { From 6f0cfcd23e91032addefe1df2d51fb35fd11cc69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Wed, 5 Aug 2026 13:15:08 +0000 Subject: [PATCH 23/27] Pack the k-mer and candidate-edge bucket formats K-mer records 24 -> ~14 B, candidate edges 17 -> ~7 B, framed with a magic, record count, length and checksum so a torn tail is recognisable. Adds --raw-records, which writes the old fixed-width form as an exactness control, and --write-header-db, since nothing between createdb and the final TSV reads the header database. createrepdb now writes the pass-2 sub-database's length-rank table. Peak scratch falls to 0.67x the previous implementation at 10M and 100M. --- data/workflow/linclustparallel.sh | 13 +- src/commons/Parameters.cpp | 7 + src/commons/Parameters.h | 4 + src/linclust/CandidateEdge.cpp | 229 +++++++++++++++-- src/linclust/CandidateEdge.h | 76 +++++- src/linclust/KmerPartition.cpp | 371 ++++++++++++++++++++++++--- src/linclust/KmerPartition.h | 150 ++++++++++- src/linclust/createrepdb.cpp | 73 +++++- src/linclust/kmermatcherparallel.cpp | 24 +- src/linclust/kmerreduceparallel.cpp | 80 +++--- src/test/TestKmerPartition.cpp | 101 ++++++-- src/util/createdbparallel.cpp | 35 ++- 12 files changed, 1021 insertions(+), 142 deletions(-) diff --git a/data/workflow/linclustparallel.sh b/data/workflow/linclustparallel.sh index 9b8c6301a..adea715fa 100755 --- a/data/workflow/linclustparallel.sh +++ b/data/workflow/linclustparallel.sh @@ -45,6 +45,12 @@ [ -z "$EVAL" ] && EVAL=0.001 [ -z "$SPLIT_MEMORY_LIMIT" ] && SPLIT_MEMORY_LIMIT=0 [ -z "$SCRATCH_BUDGET" ] && SCRATCH_BUDGET=0 +# Writes the k-mer and candidate-edge buckets as fixed-width structs instead of +# the packed block encoding. Roughly doubles the scratch those two intermediates +# need and exists only as a control: a run with RAW_RECORDS=1 must produce +# byte-identical output to one without, which is what separates an encoding +# defect from a semantic one. +[ -z "$RAW_RECORDS" ] && RAW_RECORDS=0 notExists() { [ ! -f "$1" ]; } # To stderr, not stdout: scratchUsed() runs inside a command substitution, so a @@ -176,9 +182,10 @@ mkdir -p "$TMP" KMER_COMMON="--alph-size aa:21,nucl:5 --min-seq-id $MIN_SEQ_ID --kmer-per-seq 21 \ --mask 0 --mask-prob 0.9 --mask-lower-case 0 --mask-n-repeat 0 -k 0 --max-seq-len 65535 \ --hash-shift 67 --ignore-multi-kmer 0 \ - --split-memory-limit $SPLIT_MEMORY_LIMIT --scratch-budget $SCRATCH_BUDGET --threads $THREADS" + --split-memory-limit $SPLIT_MEMORY_LIMIT --scratch-budget $SCRATCH_BUDGET \ + --raw-records $RAW_RECORDS --threads $THREADS" REDUCE_PAR="-c $COV --cov-mode $COV_MODE --include-adjacency 1 --num-adjacency 3 \ - --split-memory-limit $SPLIT_MEMORY_LIMIT --threads $THREADS" + --split-memory-limit $SPLIT_MEMORY_LIMIT --raw-records $RAW_RECORDS --threads $THREADS" ALIGN_PAR="--min-seq-id $MIN_SEQ_ID --min-aln-len 0 --seq-id-mode 0 -e $EVAL -c $COV \ --cov-mode $COV_MODE --threads $THREADS" @@ -212,7 +219,7 @@ if notExists "$INPUT.dbtype"; then # the database with `mmseqs createdbparallel --write-text-index 1` if a # stock MMseqs2 tool has to open it afterwards. $WORKER_RUNNER "$MMSEQS" createdbparallel "$INPUT" "$DB" $VERBOSITY --threads $THREADS \ - --write-text-index 0 \ + --write-text-index 0 --write-header-db 0 \ || fail "createdbparallel died" fi fi diff --git a/src/commons/Parameters.cpp b/src/commons/Parameters.cpp index d9f4e614b..5c70fed6a 100644 --- a/src/commons/Parameters.cpp +++ b/src/commons/Parameters.cpp @@ -229,6 +229,8 @@ Parameters::Parameters(): PARAM_SHUFFLE(PARAM_SHUFFLE_ID, "--shuffle", "Shuffle input database", "Shuffle input database", typeid(bool), (void *) &shuffleDatabase, ""), PARAM_WRITE_LOOKUP(PARAM_WRITE_LOOKUP_ID, "--write-lookup", "Write lookup file", "write .lookup file containing mapping from internal id, fasta id and file number", typeid(int), (void *) &writeLookup, "^[0-1]{1}", MMseqsParameter::COMMAND_EXPERT), PARAM_WRITE_TEXT_INDEX(PARAM_WRITE_TEXT_INDEX_ID, "--write-text-index", "Write text index", "Also write the stock-compatible text .index alongside the dense .index.bin. None of the distributed stages read it, and at 1e12 sequences it is ~36 TB and ~98 h of single-threaded formatting; turn it off unless a stock MMseqs2 tool has to open the database.", typeid(int), (void *) &writeTextIndex, "^[0-1]{1}$", MMseqsParameter::COMMAND_EXPERT), + PARAM_RAW_RECORDS(PARAM_RAW_RECORDS_ID, "--raw-records", "Uncompacted bucket records", "Write k-mer and candidate-edge buckets as fixed-width structs instead of the packed block encoding. Roughly doubles the scratch these intermediates need, and exists only to separate an encoding defect from a semantic one: a run with this on must produce exactly the same clustering as a run with it off.", typeid(int), (void *) &rawRecords, "^[0-1]{1}$", MMseqsParameter::COMMAND_EXPERT), + PARAM_WRITE_HEADER_DB(PARAM_WRITE_HEADER_DB_ID, "--write-header-db", "Write header database", "Also write the _h header database. Nothing between createdb and the final TSV reads it -- accessions reach the output through .lookup -- and at 1e12 sequences it is ~35 TB of a ~1 PB scratch budget. Turning it off produces a database stock MMseqs2 tools cannot open.", typeid(int), (void *) &writeHeaderDb, "^[0-1]{1}$", MMseqsParameter::COMMAND_EXPERT), PARAM_USE_HEADER_FILE(PARAM_USE_HEADER_FILE_ID, "--use-header-file", "Use header DB", "use the sequence header DB instead of the body to map the entry keys", typeid(bool), (void *) &useHeaderFile, ""), // setextendeddbtype PARAM_EXTENDED_DBTYPE(PARAM_EXTENDED_DBTYPE_ID, "--extended-dbtype", "Extended dbtype", "Set extended dbtype 1: compressed, 2: need src, 4: context pseudoe cnts", typeid(int), (void *) &extendedDbtype, "^[0-4]{1}"), @@ -924,6 +926,7 @@ Parameters::Parameters(): createdbparallel.push_back(&PARAM_DB_TYPE); createdbparallel.push_back(&PARAM_CHUNK_SIZE); createdbparallel.push_back(&PARAM_WRITE_TEXT_INDEX); + createdbparallel.push_back(&PARAM_WRITE_HEADER_DB); createdbparallel.push_back(&PARAM_THREADS); // No PARAM_COMPRESSED. emitChunk pwrites raw sequence and header bytes into // preallocated files at offsets a plan derived from *uncompressed* lengths, @@ -942,6 +945,7 @@ Parameters::Parameters(): // the grouping and alignment parameters belong to the reduce that reads them. // No PARAM_ADJUST_KMER_LEN either -- the adjusted length is a property of the // whole database that a worker scanning one key range cannot agree on. + kmermatcherparallel.push_back(&PARAM_RAW_RECORDS); kmermatcherparallel.push_back(&PARAM_SUB_MAT); kmermatcherparallel.push_back(&PARAM_ALPH_SIZE); kmermatcherparallel.push_back(&PARAM_MIN_SEQ_ID); @@ -969,6 +973,7 @@ Parameters::Parameters(): // Grouping knobs. No k-mer extraction parameters: k and the partitioning are // fixed by the shuffle manifest the map wrote, not re-derived here, so passing // them would only create a way to disagree with what is on disk. + kmerreduceparallel.push_back(&PARAM_RAW_RECORDS); kmerreduceparallel.push_back(&PARAM_SUB_MAT); kmerreduceparallel.push_back(&PARAM_ALPH_SIZE); kmerreduceparallel.push_back(&PARAM_C); @@ -2654,6 +2659,8 @@ void Parameters::setDefaults() { scratchUsed = 0; spillPrefix = ""; writeTextIndex = 1; + rawRecords = 0; + writeHeaderDb = 1; diskSpaceLimit = 0; splitAA = false; spacedKmerPattern = ""; diff --git a/src/commons/Parameters.h b/src/commons/Parameters.h index 940a7725c..9ef98bc39 100644 --- a/src/commons/Parameters.h +++ b/src/commons/Parameters.h @@ -446,6 +446,8 @@ class Parameters { size_t scratchUsed; // Bytes of the budget already occupied when a stage starts std::string spillPrefix; // translatekeys: where its spill files go, when not beside the output int writeTextIndex; // createdbparallel: also write the stock-compatible text .index + int rawRecords; // kmermatcherparallel/kmerreduceparallel: write uncompacted fixed-width records + int writeHeaderDb; // createdbparallel: also write the _h header database size_t diskSpaceLimit; // Maximum disk space in bytes for sliced reverse profile search bool splitAA; // Split database by amino acid count instead int preloadMode; // Preload mode of database @@ -1030,6 +1032,8 @@ class Parameters { PARAMETER(PARAM_SHUFFLE) PARAMETER(PARAM_WRITE_LOOKUP) PARAMETER(PARAM_WRITE_TEXT_INDEX) + PARAMETER(PARAM_RAW_RECORDS) + PARAMETER(PARAM_WRITE_HEADER_DB) // convert2fasta PARAMETER(PARAM_USE_HEADER_FILE) diff --git a/src/linclust/CandidateEdge.cpp b/src/linclust/CandidateEdge.cpp index 937198c26..fcfc990b5 100644 --- a/src/linclust/CandidateEdge.cpp +++ b/src/linclust/CandidateEdge.cpp @@ -1,5 +1,7 @@ #include "CandidateEdge.h" +#include "VarintCodec.h" + #include "Debug.h" #include "FileUtil.h" #include "Util.h" @@ -152,11 +154,188 @@ void EdgeBucketWriter::createLayout(const std::string &dir, unsigned int bucketC } } +// C++14 has no inline variables, so the in-class initialiser is not a definition. +const uint64_t CandidateEdge::MAX_KEY; + +namespace EdgeBlockCodec { + +uint32_t checksum(const uint8_t *data, size_t size) { + uint32_t hash = 2166136261U; + for (size_t i = 0; i < size; i++) { + hash ^= data[i]; + hash *= 16777619U; + } + return hash; +} + +bool headerLooksValid(const EdgeBlockHeader &header) { + if (header.magic != EdgeBlockHeader::MAGIC) { + return false; + } + if (header.encoding != ENCODING_RAW && header.encoding != ENCODING_PACKED) { + return false; + } + if (header.encoding == ENCODING_RAW && + header.payloadBytes != static_cast(header.recordCount) * sizeof(CandidateEdge)) { + return false; + } + return true; +} + +void encode(const std::vector &edges, bool raw, uint32_t partition, uint32_t worker, + std::vector &out) { + if (edges.empty()) { + return; + } + + EdgeBlockHeader header; + header.magic = EdgeBlockHeader::MAGIC; + header.partition = partition; + header.worker = worker; + header.recordCount = static_cast(edges.size()); + header.reserved[0] = header.reserved[1] = header.reserved[2] = 0; + header.reserved2 = 0; + + const size_t headerAt = out.size(); + out.resize(headerAt + sizeof(EdgeBlockHeader)); + const size_t payloadAt = out.size(); + + if (raw) { + header.encoding = ENCODING_RAW; + header.payloadBytes = static_cast(edges.size()) * sizeof(CandidateEdge); + out.resize(payloadAt + header.payloadBytes); + memcpy(&out[payloadAt], edges.data(), static_cast(header.payloadBytes)); + } else { + header.encoding = ENCODING_PACKED; + + // Checked rather than assumed. The reduce sorts by (rep, member, + // diagonal, strand) before appending and a bucket is a rep range, so a + // block is sorted by construction -- but a rep that went backwards would + // wrap its delta into a huge unsigned value and decode as a different + // edge, silently. + for (size_t i = 1; i < edges.size(); i++) { + if (edges[i].getRep() < edges[i - 1].getRep()) { + Debug(Debug::ERROR) << "Edge block for partition " << partition + << " is not sorted by representative at record " << i + << ". The packed encoding requires the order the reduce " + "produces.\n"; + EXIT(EXIT_FAILURE); + } + } + + // Sized exactly, then written. Reserving the worst case instead would put + // this buffer at 26 B per edge alongside the 17 B edge buffer it encodes, + // which is nearly twice what the writer's budget was told to expect. + size_t payloadSize = 0; + uint64_t prevRep = 0; + for (size_t i = 0; i < edges.size(); i++) { + const uint64_t rep = edges[i].getRep(); + const uint64_t member = edges[i].getMember(); + payloadSize += VarintCodec::size(rep - prevRep); + payloadSize += VarintCodec::size(VarintCodec::zigzag( + static_cast(member) - static_cast(rep))); + payloadSize += VarintCodec::size(VarintCodec::zigzag(edges[i].diagonal)); + payloadSize += VarintCodec::size((static_cast(edges[i].score) << 1) | + (edges[i].reverseStrand ? 1U : 0U)); + prevRep = rep; + } + out.resize(payloadAt + payloadSize); + uint8_t *cursor = &out[payloadAt]; + + prevRep = 0; + for (size_t i = 0; i < edges.size(); i++) { + const uint64_t rep = edges[i].getRep(); + const uint64_t member = edges[i].getMember(); + cursor = VarintCodec::write(cursor, rep - prevRep); + cursor = VarintCodec::write( + cursor, VarintCodec::zigzag(static_cast(member) - + static_cast(rep))); + cursor = VarintCodec::write(cursor, VarintCodec::zigzag(edges[i].diagonal)); + cursor = VarintCodec::write(cursor, (static_cast(edges[i].score) << 1) | + (edges[i].reverseStrand ? 1U : 0U)); + prevRep = rep; + } + header.payloadBytes = static_cast(cursor - &out[payloadAt]); + if (header.payloadBytes != payloadSize) { + Debug(Debug::ERROR) << "Edge block encoder wrote " << header.payloadBytes + << " bytes where it sized " << payloadSize << "\n"; + EXIT(EXIT_FAILURE); + } + } + + header.checksum = checksum(&out[payloadAt], static_cast(header.payloadBytes)); + memcpy(&out[headerAt], &header, sizeof(EdgeBlockHeader)); +} + +bool decode(const EdgeBlockHeader &header, const uint8_t *payload, + std::vector &out) { + if (checksum(payload, static_cast(header.payloadBytes)) != header.checksum) { + return false; + } + const size_t at = out.size(); + if (header.encoding == ENCODING_RAW) { + if (header.payloadBytes != static_cast(header.recordCount) * sizeof(CandidateEdge)) { + return false; + } + out.resize(at + header.recordCount); + memcpy(&out[at], payload, static_cast(header.payloadBytes)); + return true; + } + + const uint8_t *cursor = payload; + const uint8_t *end = payload + header.payloadBytes; + uint64_t prevRep = 0; + out.reserve(at + header.recordCount); + for (uint32_t i = 0; i < header.recordCount; i++) { + uint64_t repDelta = 0; + uint64_t memberZigzag = 0; + uint64_t diagonalZigzag = 0; + uint64_t scoreStrand = 0; + if (VarintCodec::read(cursor, end, repDelta) == false || + VarintCodec::read(cursor, end, memberZigzag) == false || + VarintCodec::read(cursor, end, diagonalZigzag) == false || + VarintCodec::read(cursor, end, scoreStrand) == false) { + out.resize(at); + return false; + } + prevRep += repDelta; + const int64_t memberOffset = VarintCodec::unzigzag(memberZigzag); + const int64_t member = static_cast(prevRep) + memberOffset; + const int64_t diagonal = VarintCodec::unzigzag(diagonalZigzag); + if (prevRep > CandidateEdge::MAX_KEY || member < 0 || + static_cast(member) > CandidateEdge::MAX_KEY || + diagonal < INT16_MIN || diagonal > INT16_MAX || (scoreStrand >> 1) > UINT16_MAX) { + out.resize(at); + return false; + } + + CandidateEdge edge; + edge.setRep(prevRep); + edge.setMember(static_cast(member)); + edge.diagonal = static_cast(diagonal); + edge.reverseStrand = static_cast(scoreStrand & 1U); + edge.score = static_cast(scoreStrand >> 1); + out.push_back(edge); + } + if (cursor != end) { + out.resize(at); + return false; + } + return true; +} + +} // namespace EdgeBlockCodec + EdgeBucketWriter::EdgeBucketWriter(const std::string &dir, unsigned int bucketCount, - const std::string &shardId, size_t bufferBudgetBytes) + const std::string &shardId, size_t bufferBudgetBytes, + bool rawRecords) : dir(dir), shardId(shardId), bucketCount(bucketCount), edgeCount(0), closed(false), - currentPartition(0), currentWorker(-1) { - const size_t perBucket = bufferBudgetBytes / (bucketCount * sizeof(CandidateEdge)); + currentPartition(0), currentWorker(-1), rawRecords(rawRecords) { + // Divided by both buffers a bucket holds: the fixed-width edges and the + // encoded block they are packed into. + const size_t perBucket = + bufferBudgetBytes / + (bucketCount * (sizeof(CandidateEdge) + EdgeBlockCodec::MAX_ENCODED_BYTES_PER_EDGE)); edgesPerBuffer = std::max(perBucket, 64); buffers.resize(bucketCount); // Reserved up front, so a std::vector's geometric growth cannot leave the @@ -164,6 +343,7 @@ EdgeBucketWriter::EdgeBucketWriter(const std::string &dir, unsigned int bucketCo for (unsigned int b = 0; b < bucketCount; b++) { buffers[b].reserve(edgesPerBuffer); } + encodeBuffers.resize(bucketCount); files.assign(bucketCount, NULL); descriptorBudget = deriveEdgeDescriptorBudget(bucketCount); openFiles = 0; @@ -204,17 +384,12 @@ void EdgeBucketWriter::flush(unsigned int bucket) { << "the partition and worker producing them.\n"; EXIT(EXIT_FAILURE); } - EdgeBlockHeader header; - header.magic = EdgeBlockHeader::MAGIC; - header.partition = currentPartition; - header.worker = static_cast(currentWorker); - header.recordCount = static_cast(buffer.size()); - if (fwrite(&header, sizeof(EdgeBlockHeader), 1, files[bucket]) != 1) { - Debug(Debug::ERROR) << "Cannot write the block header of bucket " << bucket << " of " << dir - << ": " << strerror(errno) << "\n"; - EXIT(EXIT_FAILURE); - } - if (fwrite(buffer.data(), sizeof(CandidateEdge), buffer.size(), files[bucket]) != buffer.size()) { + std::vector &encoded = encodeBuffers[bucket]; + encoded.clear(); + EdgeBlockCodec::encode(buffer, rawRecords, currentPartition, + static_cast(currentWorker), encoded); + if (encoded.empty() == false && + fwrite(encoded.data(), 1, encoded.size(), files[bucket]) != encoded.size()) { Debug(Debug::ERROR) << "Cannot write " << buffer.size() << " edges to bucket " << bucket << " of " << dir << ": " << strerror(errno) << "\n"; EXIT(EXIT_FAILURE); @@ -346,6 +521,7 @@ size_t EdgeBucketReader::readShard(const std::string &path, const std::vector payload; while (offset + sizeof(EdgeBlockHeader) <= bytes) { EdgeBlockHeader header; if (fread(&header, sizeof(EdgeBlockHeader), 1, file) != 1) { @@ -358,12 +534,12 @@ size_t EdgeBucketReader::readShard(const std::string &path, const std::vector(header.recordCount) * sizeof(CandidateEdge); // A worker killed mid-write leaves a partial block, always at the end of // its own shard: it appends, and a restarted worker takes a new id and so // a new shard. Stopping here discards exactly that tail. The partition it // belonged to is redone by another worker, whose copy is complete. - if (header.magic != EdgeBlockHeader::MAGIC || offset + blockBytes > bytes) { + if (EdgeBlockCodec::headerLooksValid(header) == false || + offset + header.payloadBytes > bytes) { Debug(Debug::WARNING) << "Edge shard " << path << " ends in a partial block at byte " << offset << "; it was written by an interrupted worker and the " << "partition it held was redone.\n"; @@ -387,24 +563,31 @@ size_t EdgeBucketReader::readShard(const std::string &path, const std::vector(blockBytes), SEEK_CUR) != 0) { + if (fseeko(file, static_cast(header.payloadBytes), SEEK_CUR) != 0) { Debug(Debug::ERROR) << "Cannot skip a superseded block in " << path << "\n"; EXIT(EXIT_FAILURE); } - offset += blockBytes; + offset += static_cast(header.payloadBytes); continue; } if (header.recordCount > 0) { - const size_t at = out.size(); - out.resize(at + header.recordCount); - if (fread(out.data() + at, sizeof(CandidateEdge), header.recordCount, file) != - header.recordCount) { + payload.resize(static_cast(header.payloadBytes)); + if (payload.empty() == false && + fread(payload.data(), 1, payload.size(), file) != payload.size()) { Debug(Debug::ERROR) << "Cannot read a block of edge shard " << path << "\n"; EXIT(EXIT_FAILURE); } + if (EdgeBlockCodec::decode(header, payload.data(), out) == false) { + // Distinguished from the partial-block case above: the bytes are + // all present but do not decode, which means the shard is not what + // its headers claim rather than merely truncated. + Debug(Debug::ERROR) << "Edge shard " << path << " has a block at byte " << offset + << " that does not decode. The shard is corrupt.\n"; + EXIT(EXIT_FAILURE); + } kept += header.recordCount; } - offset += blockBytes; + offset += static_cast(header.payloadBytes); } fclose(file); return kept; diff --git a/src/linclust/CandidateEdge.h b/src/linclust/CandidateEdge.h index 9f0cd2126..39e519665 100644 --- a/src/linclust/CandidateEdge.h +++ b/src/linclust/CandidateEdge.h @@ -6,6 +6,8 @@ #include #include +#include "VarintCodec.h" + // One (representative, member) pair proposed by the distributed reduce. // // Packed binary rather than a prefilter DB. A `pref` DB stores each hit as ASCII @@ -57,6 +59,10 @@ struct __attribute__((__packed__)) CandidateEdge { // can still disagree on a pair whose per-entry count exceeds 255. uint16_t score; + // 48-bit keys, matching KmerRecord::MAX_ID. Checked on decode: a packed edge + // whose delta arithmetic produced a key past this is corrupt, not large. + static const uint64_t MAX_KEY = (static_cast(1) << 48) - 1; + uint64_t getRep() const { return get(repBytes); } uint64_t getMember() const { return get(memberBytes); } void setRep(uint64_t key) { set(repBytes, key); } @@ -139,10 +145,74 @@ struct __attribute__((__packed__)) EdgeBlockHeader { uint32_t partition; uint32_t worker; uint32_t recordCount; + // ENCODING_RAW or ENCODING_PACKED. Named per block, so --raw-records is a + // writer-side switch that costs the reader nothing and a directory holding + // both kinds still reads correctly. + uint8_t encoding; + uint8_t reserved[3]; + uint64_t payloadBytes; + // FNV-1a over the payload. Length alone catches a truncated block; this also + // catches a block boundary that has drifted, which with variable-width + // records would otherwise decode into plausible nonsense. + uint32_t checksum; + uint32_t reserved2; static const uint32_t MAGIC = 0x45444745; // "EDGE" }; +// Packs a sorted run of edges. +// +// This is the intermediate that decides whether the pipeline fits its scratch +// budget. Waves divide the k-mer shuffle but not the edges -- alignment runs once, +// after every wave -- so at 1e12 the edges are the term nothing else reduces: +// ~22 records per sequence measured at 1e9, at 17 fixed-width bytes each. +// +// Four fields, each encoded against what the surrounding design already +// guarantees: +// +// - `rep` as a delta. The reduce sorts its edges by (rep, member, diagonal, +// strand) and appends them in that order, and bucket == rep / bucketSpan, so +// a bucket's appends are a subsequence of a sorted sequence and every block is +// sorted by construction. No sort is needed here. +// - `member` as a zigzag delta *from its own rep*, not from the previous +// member. Keys are length-ranked and homologous sequences have similar +// lengths, so |rep - member| is small for most edges -- 82.5% below 1e7 on +// measured UniRef100 data. This is a direct dividend of the length-ranked key +// design; with input-order keys it would be a full 6 bytes. +// - `diagonal` zigzagged. It stays an int16_t: stock truncates an int into a +// short (kmermatcher.cpp:2050) and DistanceCalculator undoes it by trying +// every diagonal congruent mod 65536, so widening it would diverge from stock +// rather than converge on it. +// - `score` and `reverseStrand` share a varint, strand in the low bit. Strand +// belongs in the sort key (two opposite-strand edges on one diagonal are +// different alignments), but it is one bit and a byte of its own would be +// 6% of the record. +namespace EdgeBlockCodec { + +const uint8_t ENCODING_RAW = 0; +const uint8_t ENCODING_PACKED = 1; + +// Ceiling on the bytes one edge can encode to, used to size the writer's budget +// against both the edge buffer and the encode buffer it packs into. +const size_t MAX_ENCODED_BYTES_PER_EDGE = VarintCodec::MAX_BYTES * 2 + 3 + 3; + +// Appends one framed block. Requires edges sorted by non-decreasing rep, which +// the reduce guarantees; the encoder checks rather than assumes, because a +// negative delta would wrap into a huge one and decode into a different edge. +void encode(const std::vector &edges, bool raw, uint32_t partition, uint32_t worker, + std::vector &out); + +// Decodes one block payload, appending to out. Returns false on corruption rather +// than exiting, so a caller can stop at the last good block. +bool decode(const EdgeBlockHeader &header, const uint8_t *payload, + std::vector &out); + +bool headerLooksValid(const EdgeBlockHeader &header); + +uint32_t checksum(const uint8_t *data, size_t size); + +} // namespace EdgeBlockCodec + // Writes edges into buckets by *representative key range*. // // This is the layout the alignment stage needs, and the reason is not obvious: @@ -164,7 +234,7 @@ struct __attribute__((__packed__)) EdgeBlockHeader { class EdgeBucketWriter { public: EdgeBucketWriter(const std::string &dir, unsigned int bucketCount, const std::string &shardId, - size_t bufferBudgetBytes = 256 * 1024 * 1024); + size_t bufferBudgetBytes = 256 * 1024 * 1024, bool rawRecords = false); ~EdgeBucketWriter(); // Names the partition every subsequent append belongs to, and the worker @@ -197,6 +267,8 @@ class EdgeBucketWriter { unsigned int bucketCount; size_t edgesPerBuffer; std::vector > buffers; + // One reusable encode target per bucket, so a flush allocates nothing. + std::vector > encodeBuffers; std::vector files; // Descriptors kept open across flushes, up to descriptorBudget. Plain counters: // one writer belongs to one process and its appends come from the single @@ -207,6 +279,8 @@ class EdgeBucketWriter { bool closed; unsigned int currentPartition; int64_t currentWorker; + // Writes fixed-width edges instead of packed blocks; see --raw-records. + bool rawRecords; }; // Reads the blocks EdgeBucketWriter wrote, keeping only those whose producer the diff --git a/src/linclust/KmerPartition.cpp b/src/linclust/KmerPartition.cpp index f9a244b67..8add91a59 100644 --- a/src/linclust/KmerPartition.cpp +++ b/src/linclust/KmerPartition.cpp @@ -3,9 +3,11 @@ #include "Debug.h" #include "FileUtil.h" #include "Util.h" +#include "VarintCodec.h" #include #include +#include #include #include @@ -178,6 +180,296 @@ std::string KmerBucketWriter::partitionDir(const std::string &dir, unsigned int return dir + "/p" + SSTR(partition); } +namespace KmerBlockCodec { + +uint32_t checksum(const uint8_t *data, size_t size) { + // FNV-1a. Chosen for being a few instructions per byte on data that is + // already in cache from having just been written or read; this is not a + // cryptographic check, it is a guard against a block boundary that has drifted. + uint32_t hash = 2166136261U; + for (size_t i = 0; i < size; i++) { + hash ^= data[i]; + hash *= 16777619U; + } + return hash; +} + +bool headerLooksValid(const KmerBlockHeader &header) { + if (header.magic != MAGIC) { + return false; + } + if (header.encoding != ENCODING_RAW && header.encoding != ENCODING_PACKED) { + return false; + } + if (header.encoding == ENCODING_PACKED && (header.kmerWidth < 1 || header.kmerWidth > 8)) { + return false; + } + if (header.encoding == ENCODING_RAW && + header.payloadBytes != header.recordCount * sizeof(KmerRecord)) { + return false; + } + return true; +} + +// Six residues at five bits each, little-endian within a 32-bit word. +static uint32_t packAdjacent(const uint8_t *adjacent) { + uint32_t bits = 0; + for (unsigned int i = 0; i < ADJACENT_COUNT; i++) { + uint32_t value = adjacent[i]; + if (value == UCHAR_MAX) { + value = ADJACENT_SENTINEL; + } else if (value >= ADJACENT_SENTINEL) { + // Only reachable if the alphabet is wider than the packing allows, + // which is checked once at startup. Reaching it here means that check + // was skipped, and silently truncating would corrupt the adjacency + // rounds rather than fail. + Debug(Debug::ERROR) << "Adjacent residue index " << value + << " does not fit the 5-bit packing (max " + << (ADJACENT_SENTINEL - 1) << ", or " << UCHAR_MAX + << " for absent)\n"; + EXIT(EXIT_FAILURE); + } + bits |= value << (i * ADJACENT_BITS); + } + return bits; +} + +static void unpackAdjacent(uint32_t bits, uint8_t *adjacent) { + for (unsigned int i = 0; i < ADJACENT_COUNT; i++) { + const uint32_t value = (bits >> (i * ADJACENT_BITS)) & 0x1FU; + adjacent[i] = (value == ADJACENT_SENTINEL) ? UCHAR_MAX : static_cast(value); + } +} + +void encode(std::vector &records, bool raw, std::vector &out) { + if (records.empty()) { + return; + } + + KmerBlockHeader header; + header.magic = MAGIC; + header.reserved = 0; + header.reserved2 = 0; + header.recordCount = records.size(); + + const size_t headerAt = out.size(); + out.resize(headerAt + sizeof(KmerBlockHeader)); + const size_t payloadAt = out.size(); + + if (raw) { + header.encoding = ENCODING_RAW; + header.kmerWidth = 0; + header.payloadBytes = records.size() * sizeof(KmerRecord); + out.resize(payloadAt + header.payloadBytes); + memcpy(&out[payloadAt], records.data(), header.payloadBytes); + } else { + // Ascending ids are what make the id delta a byte; see the note on + // encode() in the header. Ties broken by (kmer, pos) so the encoding is a + // pure function of the record multiset, which is what lets a redone work + // item produce byte-identical output. + std::sort(records.begin(), records.end(), + [](const KmerRecord &a, const KmerRecord &b) { + const uint64_t ida = a.getId(); + const uint64_t idb = b.getId(); + if (ida != idb) { + return ida < idb; + } + if (a.kmer != b.kmer) { + return a.kmer < b.kmer; + } + return a.pos < b.pos; + }); + + uint64_t maxKmer = 0; + for (size_t i = 0; i < records.size(); i++) { + maxKmer = std::max(maxKmer, records[i].kmer); + } + const unsigned int kmerWidth = VarintCodec::fixedWidthFor(maxKmer); + header.encoding = ENCODING_PACKED; + header.kmerWidth = static_cast(kmerWidth); + + // Sized exactly rather than reserved at the worst case. The loose bound is + // 25 B per record against a ~13.5 B result, and this buffer is live + // alongside the 24 B record buffer it encodes -- so reserving the bound + // would put the writer's real footprint at nearly twice what the budget + // was told to expect. Costing one extra pass over data already in cache is + // the cheaper half of that trade. + size_t payloadSize = 0; + { + uint64_t prevId = 0; + for (size_t i = 0; i < records.size(); i++) { + const uint64_t id = records[i].getId(); + payloadSize += VarintCodec::size(id - prevId); + payloadSize += VarintCodec::size(records[i].pos); + prevId = id; + } + payloadSize += records.size() * (kmerWidth + ADJACENT_BYTES); + } + out.resize(payloadAt + payloadSize); + uint8_t *cursor = &out[payloadAt]; + + // prevId starts at zero, so the first record's "delta" is its absolute id. + // That keeps the decode loop uniform and costs a few bytes once per block. + uint64_t prevId = 0; + for (size_t i = 0; i < records.size(); i++) { + const uint64_t id = records[i].getId(); + cursor = VarintCodec::write(cursor, id - prevId); + prevId = id; + cursor = VarintCodec::writeFixed(cursor, records[i].kmer, kmerWidth); + cursor = VarintCodec::write(cursor, records[i].pos); + cursor = VarintCodec::writeFixed(cursor, packAdjacent(records[i].adjacent), + ADJACENT_BYTES); + } + header.payloadBytes = static_cast(cursor - &out[payloadAt]); + if (header.payloadBytes != payloadSize) { + Debug(Debug::ERROR) << "K-mer block encoder wrote " << header.payloadBytes + << " bytes where it sized " << payloadSize << "\n"; + EXIT(EXIT_FAILURE); + } + } + + header.checksum = checksum(&out[payloadAt], header.payloadBytes); + memcpy(&out[headerAt], &header, sizeof(KmerBlockHeader)); +} + +bool decode(const KmerBlockHeader &header, const uint8_t *payload, + const LengthRankTable *lengths, std::vector &out) { + if (checksum(payload, header.payloadBytes) != header.checksum) { + return false; + } + if (header.encoding == ENCODING_RAW) { + if (header.payloadBytes != header.recordCount * sizeof(KmerRecord)) { + return false; + } + const size_t at = out.size(); + out.resize(at + header.recordCount); + memcpy(&out[at], payload, header.payloadBytes); + return true; + } + + // Packed blocks do not carry seqLen; it comes from the key. + if (lengths == NULL || lengths->isOpen() == false) { + Debug(Debug::ERROR) << "A packed k-mer block needs the length-rank table to recover " + << "sequence lengths, and none was opened\n"; + EXIT(EXIT_FAILURE); + } + + const uint8_t *cursor = payload; + const uint8_t *end = payload + header.payloadBytes; + uint64_t prevId = 0; + const size_t at = out.size(); + out.reserve(at + header.recordCount); + for (uint64_t i = 0; i < header.recordCount; i++) { + uint64_t idDelta = 0; + uint64_t kmer = 0; + uint64_t pos = 0; + uint64_t adjacentBits = 0; + if (VarintCodec::read(cursor, end, idDelta) == false || + VarintCodec::readFixed(cursor, end, header.kmerWidth, kmer) == false || + VarintCodec::read(cursor, end, pos) == false || + VarintCodec::readFixed(cursor, end, ADJACENT_BYTES, adjacentBits) == false) { + out.resize(at); + return false; + } + prevId += idDelta; + if (prevId > KmerRecord::MAX_ID || pos > UINT16_MAX) { + out.resize(at); + return false; + } + unsigned int length = 0; + if (lengths->tryLengthOf(prevId, length) == false) { + out.resize(at); + return false; + } + + KmerRecord record; + record.kmer = kmer; + record.setId(prevId); + record.pos = static_cast(pos); + record.seqLen = static_cast(std::min(length, UINT16_MAX)); + unpackAdjacent(static_cast(adjacentBits), record.adjacent); + out.push_back(record); + } + // Trailing bytes mean the block is not what its header says it is. + if (cursor != end) { + out.resize(at); + return false; + } + return true; +} + +} // namespace KmerBlockCodec + +KmerShardReader::KmerShardReader(const std::string &path) : path(path), torn(false) { + file = fopen(path.c_str(), "rb"); + if (file == NULL) { + // A shard that vanished under us is not an error: the reduce unlinks + // consumed partitions, and a restart can race that sweep. + torn = false; + } +} + +KmerShardReader::~KmerShardReader() { + if (file != NULL) { + fclose(file); + } +} + +bool KmerShardReader::next(std::vector &out, const LengthRankTable *lengths) { + if (file == NULL) { + return false; + } + KmerBlockHeader header; + const size_t got = fread(&header, 1, sizeof(KmerBlockHeader), file); + if (got == 0) { + return false; // clean end of shard + } + if (got != sizeof(KmerBlockHeader) || KmerBlockCodec::headerLooksValid(header) == false) { + torn = true; + return false; + } + payload.resize(static_cast(header.payloadBytes)); + if (header.payloadBytes > 0 && + fread(payload.data(), 1, payload.size(), file) != payload.size()) { + torn = true; + return false; + } + if (KmerBlockCodec::decode(header, payload.data(), lengths, out) == false) { + torn = true; + return false; + } + return true; +} + +uint64_t KmerShardReader::countRecords(const std::string &path, bool &torn) { + torn = false; + FILE *file = fopen(path.c_str(), "rb"); + if (file == NULL) { + return 0; + } + uint64_t total = 0; + while (true) { + KmerBlockHeader header; + const size_t got = fread(&header, 1, sizeof(KmerBlockHeader), file); + if (got == 0) { + break; + } + if (got != sizeof(KmerBlockHeader) || KmerBlockCodec::headerLooksValid(header) == false) { + torn = true; + break; + } + // Seeking past the payload rather than reading it is what keeps this + // O(blocks) instead of O(records). + if (fseeko(file, static_cast(header.payloadBytes), SEEK_CUR) != 0) { + torn = true; + break; + } + total += header.recordCount; + } + fclose(file); + return total; +} + void KmerBucketWriter::createLayout(const std::string &dir, unsigned int partitionCount) { if (FileUtil::directoryExists(dir.c_str()) == false) { FileUtil::makeDir(dir.c_str()); @@ -198,10 +490,11 @@ void KmerBucketWriter::createLayout(const std::string &dir, unsigned int partiti KmerBucketWriter::KmerBucketWriter(const std::string &dir, unsigned int partitionCount, const std::string &shardId, size_t bufferBudgetBytes, - unsigned int partitionFrom, unsigned int partitionTo) + unsigned int partitionFrom, unsigned int partitionTo, + bool rawRecords) : dir(dir), shardId(shardId), partitionCount(partitionCount), mutexes(partitionCount), partitionFrom(partitionFrom), - partitionTo(partitionTo == 0 ? partitionCount : partitionTo) { + partitionTo(partitionTo == 0 ? partitionCount : partitionTo), rawRecords(rawRecords) { // At least a handful of records per partition even with a tiny budget, so a // large partition count degrades to more frequent flushes rather than to // one write syscall per k-mer. @@ -213,7 +506,13 @@ KmerBucketWriter::KmerBucketWriter(const std::string &dir, unsigned int partitio // filesystem cares about. const unsigned int activeCount = (this->partitionTo > partitionFrom) ? (this->partitionTo - partitionFrom) : 1; - const size_t perPartition = bufferBudgetBytes / (activeCount * sizeof(KmerRecord)); + // Divided by both buffers a partition holds: the 24-byte records and the + // encoded block they are packed into. Sizing against the records alone would + // understate the writer's footprint by the encoded size, which is the same + // class of quiet overshoot that reserving the encoder's worst case would be. + const size_t perPartition = + bufferBudgetBytes / + (activeCount * (sizeof(KmerRecord) + KmerBlockCodec::MAX_ENCODED_BYTES_PER_RECORD)); recordsPerBuffer = std::max(perPartition, 16); buffers.resize(partitionCount); // Reserved up front: push_back until size() >= recordsPerBuffer lets a @@ -222,6 +521,7 @@ KmerBucketWriter::KmerBucketWriter(const std::string &dir, unsigned int partitio for (unsigned int p = partitionFrom; p < this->partitionTo && p < partitionCount; p++) { buffers[p].reserve(recordsPerBuffer); } + encodeBuffers.resize(partitionCount); files.assign(partitionCount, NULL); recordCounts.assign(partitionCount, 0); descriptorBudget = deriveDescriptorBudget(activeCount); @@ -269,7 +569,15 @@ void KmerBucketWriter::flush(unsigned int partition) { } openFiles.fetch_add(1); } - if (fwrite(buffer.data(), sizeof(KmerRecord), buffer.size(), files[partition]) != buffer.size()) { + // Framed and packed here rather than written as fixed-width structs. The + // encode buffer is per partition and reused across flushes, so it allocates + // once; recordsPerBuffer is derived against both buffers so the pair stays + // inside the budget. + std::vector &encoded = encodeBuffers[partition]; + encoded.clear(); + KmerBlockCodec::encode(buffer, rawRecords, encoded); + if (encoded.empty() == false && + fwrite(encoded.data(), 1, encoded.size(), files[partition]) != encoded.size()) { // Name the reason: a full scratch filesystem is by far the likeliest way // this stage fails, and "cannot write" alone sends you looking for a bug. Debug(Debug::ERROR) << "Cannot write " << buffer.size() << " k-mer records to bucket " @@ -383,60 +691,47 @@ uint64_t KmerBucketReader::countRecords(const std::string &dir, unsigned int par const std::vector shards = shardFiles(dir, partition); uint64_t total = 0; for (size_t i = 0; i < shards.size(); i++) { - const size_t bytes = FileUtil::getFileSize(shards[i]); - if (bytes % sizeof(KmerRecord) != 0) { - // Counted down to the last whole record rather than refused. A torn + // Block headers only, seeking past each payload: O(blocks) rather than + // O(records), and with variable-width records the file size no longer + // gives the count at all. + bool torn = false; + total += KmerShardReader::countRecords(shards[i], torn); + if (torn) { + // Counted down to the last whole block rather than refused. A torn // tail is exactly what an interrupted worker leaves, and the map // redoes that item into a *different* shard, so the records are not // lost -- but the torn shard stays on disk, and making it fatal meant // every later reduce of that partition died on it forever, with no - // recovery but deleting the file by hand. readPartitionAsPositions - // already stops at the last whole record for the same reason. - Debug(Debug::WARNING) << "Bucket " << shards[i] << " ends mid-record at " << bytes - << " bytes, as an interrupted worker leaves it; reading the " - << (bytes / sizeof(KmerRecord)) << " whole records it holds.\n"; + // recovery but deleting the file by hand. + Debug(Debug::WARNING) << "Bucket " << shards[i] + << " ends on a partial block, as an interrupted worker leaves " + "it; counting the whole blocks it holds.\n"; } - total += bytes / sizeof(KmerRecord); } return total; } void KmerBucketReader::readPartition(const std::string &dir, unsigned int partition, - std::vector &out) { + std::vector &out, const LengthRankTable *lengths) { const std::vector shards = shardFiles(dir, partition); // Sized in one go rather than grown per shard: resize() on a vector holding // hundreds of gigabytes reallocates and copies the whole array once per shard, // and a partition has one shard per worker. uint64_t total = 0; - std::vector counts(shards.size(), 0); for (size_t i = 0; i < shards.size(); i++) { - counts[i] = FileUtil::getFileSize(shards[i]) / sizeof(KmerRecord); - total += counts[i]; + bool torn = false; + total += KmerShardReader::countRecords(shards[i], torn); } out.reserve(out.size() + static_cast(total)); for (size_t i = 0; i < shards.size(); i++) { - // Rounded down to whole records rather than refused, matching countRecords - // and readPartitionAsPositions. A torn tail is what an interrupted worker - // leaves; its item is redone into a *different* shard, so nothing is lost, - // and making it fatal meant every later read of that partition died on it - // forever with no recovery but deleting the file by hand. - const size_t bytes = FileUtil::getFileSize(shards[i]); - if (bytes % sizeof(KmerRecord) != 0) { - Debug(Debug::WARNING) << "Bucket " << shards[i] << " ends mid-record at " << bytes - << " bytes, as an interrupted worker leaves it; reading the " - << counts[i] << " whole records it holds.\n"; - } - const size_t count = counts[i]; - if (count == 0) { - continue; + KmerShardReader reader(shards[i]); + while (reader.next(out, lengths)) { } - FILE *file = FileUtil::openFileOrDie(shards[i].c_str(), "rb", true); - const size_t offset = out.size(); - out.resize(offset + count); - if (fread(out.data() + offset, sizeof(KmerRecord), count, file) != count) { - Debug(Debug::ERROR) << "Cannot read bucket " << shards[i] << "\n"; - EXIT(EXIT_FAILURE); + if (reader.endedTorn()) { + Debug(Debug::WARNING) << "Bucket " << shards[i] + << " ends on a partial block, as an interrupted worker leaves " + "it; reading the whole blocks it holds.\n"; } - fclose(file); } } + diff --git a/src/linclust/KmerPartition.h b/src/linclust/KmerPartition.h index 410537ad6..1e5c874ea 100644 --- a/src/linclust/KmerPartition.h +++ b/src/linclust/KmerPartition.h @@ -8,6 +8,9 @@ #include #include +#include "LengthRankTable.h" +#include "VarintCodec.h" + // K-mer space partitioning for the distributed linclust map/reduce. // // Stock kmermatcher handles a database too large for memory by *splitting*: it @@ -62,7 +65,7 @@ class KmerPartitioner { unsigned int mask; }; -// One k-mer occurrence as stored in a bucket file. +// One k-mer occurrence, as held in memory. // // 24 bytes, packed. Two fields are here specifically to delete arrays that stock // kmermatcher sizes by *key space* rather than by entry count, and which @@ -70,6 +73,14 @@ class KmerPartitioner { // - seqLen removes `seqkey_to_len[dbKeySize]` (kmermatcher.cpp:1236), // - id is carried explicitly so nothing needs a dense key-indexed side table. // `countTable` (:1256) and `repSequence` (:1357) fall the same way. +// +// **This is the in-memory form, no longer the on-disk one.** Bucket files hold +// the packed block encoding below, which is ~13.5 B per record against these 24. +// `seqLen` in particular is never written: keys are length-ranked, so +// LengthRankTable recovers it from the id for the price of a binary search over a +// table of distinct lengths. Keeping the field resident costs nothing (the decode +// buffer is bounded) and keeps kmerRecordToPosition and everything downstream +// unchanged. struct __attribute__((__packed__)) KmerRecord { uint64_t kmer; // 48 bits is 2.8e14 keys, comfortably past the 1e12 target, and saves 2 bytes @@ -170,6 +181,125 @@ void kmerPositionToRecord(KmerPositionT &in, KmerRecord &out) { } } +// Framing for one flushed run of k-mer records. +// +// Three things this buys, beyond the packing itself: +// +// - a torn tail from a worker killed mid-flush is *recognisable*. The reader +// rounds down to the last whole block and carries on, which is the behaviour +// the fixed-width format could only approximate by rounding the file size +// down to a record boundary -- and could not do at all once records vary in +// width. +// - the record count is readable without decoding, so the reduce can size its +// array in one allocation by scanning headers rather than payloads. +// - the encoding is named per block, so `--raw-records` is a writer-side switch +// that costs the reader nothing: a directory holding both kinds still reads +// correctly, which is what makes the two comparable in one run. +struct __attribute__((__packed__)) KmerBlockHeader { + // Distinguishes a real header from the tail of a torn write. + uint32_t magic; + // ENCODING_RAW or ENCODING_PACKED. + uint8_t encoding; + // Bytes per k-mer field, packed mode only. Derived from the largest k-mer in + // *this* block rather than from (k, alphabetSize): it needs no parameter + // plumbing, it cannot overflow for a large nucleotide k, and it adapts when a + // block happens to hold only small k-mers. + uint8_t kmerWidth; + uint16_t reserved; + uint64_t recordCount; + uint64_t payloadBytes; + // FNV-1a over the payload, computed while the bytes are already in hand. A + // truncated block is caught by payloadBytes alone; this also catches a block + // boundary that has drifted, which would otherwise decode into plausible + // nonsense. + uint32_t checksum; + uint32_t reserved2; +}; + +namespace KmerBlockCodec { + +const uint32_t MAGIC = 0x4B424C4BU; // "KBLK" +const uint8_t ENCODING_RAW = 0; +const uint8_t ENCODING_PACKED = 1; + +// Six adjacent residues at five bits each. Stock uses UCHAR_MAX as the "no +// adjacency" sentinel that assignGroup tests, so it maps onto the one value five +// bits leaves spare. +const uint8_t ADJACENT_SENTINEL = 31; +const unsigned int ADJACENT_COUNT = 6; +const unsigned int ADJACENT_BITS = 5; +const unsigned int ADJACENT_BYTES = 4; + +// Ceiling on the bytes one record can encode to: a 10-byte id delta, an 8-byte +// k-mer, a 3-byte position and the packed adjacency. The writer sizes its buffer +// budget against this so the encode buffer and the record buffer together stay +// inside it. +const size_t MAX_ENCODED_BYTES_PER_RECORD = VarintCodec::MAX_BYTES + 8 + 3 + ADJACENT_BYTES; + +// Largest residue index the packing can carry. Checked against the alphabet at +// startup rather than per record. +const unsigned int MAX_ALPHABET_SIZE = 31; + +// Sorts records by (id, kmer, pos) and appends one framed block to out. +// +// The sort is what makes the id field cost about one byte: the map scans a +// contiguous key range, so a flush holds ids from that range and, once ordered, +// consecutive deltas are the range divided by the record count. Threads share the +// per-partition buffer (that is deliberate -- a per-thread buffer would divide the +// write size by the thread count), so the ordering has to be restored here rather +// than relied on. Measured against the whole pipeline the sort is a rounding +// error; the field it shrinks is 6 bytes on every one of 2.1e13 records at 1e12. +void encode(std::vector &records, bool raw, std::vector &out); + +// Decodes one block payload, appending to out. +// +// `lengths` may be NULL only for a raw block, which carries seqLen itself. That +// asymmetry is deliberate: it makes --raw-records a control that does not depend +// on the length table, so comparing the two paths tests the table as well as the +// codec. +// +// Returns false on a corrupt payload rather than exiting, because the caller's +// correct response is to stop at the last good block, not to fail the run. +bool decode(const KmerBlockHeader &header, const uint8_t *payload, + const LengthRankTable *lengths, std::vector &out); + +// Whether a header could plausibly start a block. Cheap pre-check before +// trusting payloadBytes. +bool headerLooksValid(const KmerBlockHeader &header); + +uint32_t checksum(const uint8_t *data, size_t size); + +} // namespace KmerBlockCodec + +// Reads framed blocks from one shard file. +class KmerShardReader { +public: + explicit KmerShardReader(const std::string &path); + ~KmerShardReader(); + + // Appends the next block's records to out. Returns false at end of file or at + // the first block that does not decode, which is how a torn tail ends the + // shard without failing the run. + bool next(std::vector &out, const LengthRankTable *lengths); + + // True if the shard ended on a partial or corrupt block rather than cleanly. + bool endedTorn() const { return torn; } + + // Sums recordCount over the shard's block headers, seeking past each payload. + // O(blocks), not O(records): a shard holds ~1e7 records in a few hundred + // blocks at the target scale, so this is a few hundred forward seeks. + static uint64_t countRecords(const std::string &path, bool &torn); + +private: + KmerShardReader(const KmerShardReader &); + KmerShardReader &operator=(const KmerShardReader &); + + FILE *file; + std::string path; + std::vector payload; + bool torn; +}; + // Appends k-mer records into per-partition bucket files. // // One writer per worker *process*, shared by all its threads, writing @@ -205,7 +335,8 @@ class KmerBucketWriter { // Appends outside the window are dropped; the wave that owns them writes them. KmerBucketWriter(const std::string &dir, unsigned int partitionCount, const std::string &shardId, size_t bufferBudgetBytes = 1024 * 1024 * 1024, - unsigned int partitionFrom = 0, unsigned int partitionTo = 0); + unsigned int partitionFrom = 0, unsigned int partitionTo = 0, + bool rawRecords = false); ~KmerBucketWriter(); // Thread-safe. @@ -244,11 +375,21 @@ class KmerBucketWriter { unsigned int partitionCount; size_t recordsPerBuffer; std::vector > buffers; + // One reusable encode target per partition, so a flush allocates nothing. + // Per partition rather than shared because flush() runs under the + // per-partition mutex and a shared buffer would serialise every partition + // behind one lock. + std::vector > encodeBuffers; std::vector files; std::vector mutexes; std::vector recordCounts; unsigned int partitionFrom; unsigned int partitionTo; // exclusive + // Writes fixed-width records instead of packed blocks. The escape hatch that + // separates a codec defect from a semantic one: a run with it on must produce + // the same candidate edges as a run with it off, and if it does not, the + // difference is in the encoding rather than in the clustering. + bool rawRecords; // Descriptors are kept open across flushes up to this many; past it a flush // reverts to open-append-close. Counted atomically because the count is shared // across the per-partition mutexes. @@ -260,7 +401,7 @@ class KmerBucketWriter { class KmerBucketReader { public: // Total records across all shards of the partition, so the reduce stage can - // size its array in one allocation before reading. + // size its array in one allocation before reading. Reads block headers only. static uint64_t countRecords(const std::string &dir, unsigned int partition); // Appends every record of the partition to out. @@ -276,8 +417,9 @@ class KmerBucketReader { // Dropping them is exact, not a heuristic: (kmer, id, pos) identifies one // k-mer occurrence in one sequence, so two byte-identical records can only // come from the same occurrence being written twice. + // lengths is required unless every block was written with --raw-records. static void readPartition(const std::string &dir, unsigned int partition, - std::vector &out); + std::vector &out, const LengthRankTable *lengths); static std::vector shardFiles(const std::string &dir, unsigned int partition); }; diff --git a/src/linclust/createrepdb.cpp b/src/linclust/createrepdb.cpp index f4a1721dc..0af8209ae 100644 --- a/src/linclust/createrepdb.cpp +++ b/src/linclust/createrepdb.cpp @@ -48,6 +48,7 @@ #include "Command.h" #include "Debug.h" #include "DenseIndex.h" +#include "LengthRankTable.h" #include "FastSort.h" #include "FileUtil.h" #include "Parameters.h" @@ -192,13 +193,16 @@ struct CopyPlan { uint32_t maxLen; CopyPlan() : totalBytes(0), maxLen(0) {} + // One entry per distinct sequence length among the representatives, + // longest first. Written as the sub-database's .lenrank. + std::vector lengthRuns; }; // One streaming pass over the source index. Optionally emits the sub-key -> // original-key map as it goes, which is what keeps that 8 B per representative // off the heap. CopyPlan planCopy(int srcIdx, const Bitmap &keep, uint64_t entryCount, uint64_t keptCount, - FILE *keyMapOut, const std::string &keyMapPath) { + FILE *keyMapOut, const std::string &keyMapPath, bool lengthRanked) { CopyPlan plan; plan.srcRow.reserve(static_cast(keptCount / CHUNK_ENTRIES + 2)); plan.dataOffset.reserve(static_cast(keptCount / CHUNK_ENTRIES + 2)); @@ -222,6 +226,38 @@ CopyPlan planCopy(int srcIdx, const Bitmap &keep, uint64_t entryCount, uint64_t } total += buf[i].length; plan.maxLen = std::max(plan.maxLen, buf[i].length); + // The sub-database's length-rank table, accumulated in the same pass. + // + // It exists because pass 2 runs on this database and every stage there + // recovers a sequence length from its key. It is free to build here: + // original keys are length-ranked, so the representatives -- visited in + // ascending original-key order -- already come out longest first, and + // the runs are just the points where the length changes. + // + // An entry occupies its residues plus a newline and DBWriter's + // terminating null, which is the same +2 createdbparallel plans with. + if (lengthRanked) { + const uint64_t residues = buf[i].length >= 2 ? buf[i].length - 2 : 0; + if (plan.lengthRuns.empty() == false && + residues > plan.lengthRuns.back().length) { + // Only reachable if the source database is not length-ranked, + // which every stage here assumes. Silently writing a + // non-monotone table would give pass 2 plausible wrong lengths. + Debug(Debug::ERROR) + << "Source key " << (from + i) << " is longer (" << residues + << ") than a representative before it (" << plan.lengthRuns.back().length + << "). The source database is not length-ranked.\n"; + EXIT(EXIT_FAILURE); + } + if (plan.lengthRuns.empty() || plan.lengthRuns.back().length != residues) { + LengthRankTable::Run run; + run.length = residues; + run.firstKey = kept; + run.count = 0; + plan.lengthRuns.push_back(run); + } + plan.lengthRuns.back().count++; + } if (keyMapOut != NULL) { keyBuf.push_back(from + i); if (keyBuf.size() == keyBuf.capacity()) { @@ -260,13 +296,18 @@ struct CopyResult { uint32_t maxLen; }; +// lengthRanked says whether the entries are sequences ordered longest first, in +// which case the sub-database gets its own length-rank table. It is false for the +// header database: an entry's "length" there is its header text, which has no +// relation to sequence length and is not monotone in the key. CopyResult copyFlagged(const std::string &srcDb, const std::string &dstDb, const Bitmap &keep, uint64_t entryCount, uint64_t keptCount, int threads, - FILE *keyMapOut, const std::string &keyMapPath) { + FILE *keyMapOut, const std::string &keyMapPath, bool lengthRanked) { const int srcIdx = openOrDie(DenseIndex::fileName(srcDb), O_RDONLY); const int srcData = openOrDie(srcDb, O_RDONLY); - const CopyPlan plan = planCopy(srcIdx, keep, entryCount, keptCount, keyMapOut, keyMapPath); + const CopyPlan plan = + planCopy(srcIdx, keep, entryCount, keptCount, keyMapOut, keyMapPath, lengthRanked); CopyResult result; result.maxLen = plan.maxLen; @@ -274,6 +315,11 @@ CopyResult copyFlagged(const std::string &srcDb, const std::string &dstDb, const allocate(dstDb, plan.totalBytes); DenseIndex::createEmpty(dstDb, keptCount, 0, plan.totalBytes, plan.maxLen); + // Pass 2 addresses this database by key and recovers lengths from the key, so + // the sub-database needs its own length-rank table just as the full one does. + if (lengthRanked) { + LengthRankTable::write(dstDb, plan.lengthRuns, keptCount); + } const int dstData = openOrDie(dstDb, O_WRONLY); const int dstIdx = openOrDie(DenseIndex::fileName(dstDb), O_WRONLY); @@ -810,7 +856,8 @@ int createrepdb(int argc, const char **argv, const Command &command) { FILE *m = FileUtil::openAndDelete(keymapTmp.c_str(), "wb"); const CopyResult seq = - copyFlagged(seqDb, repDb, isRep, info.entryCount, repCount, par.threads, m, keymapTmp); + copyFlagged(seqDb, repDb, isRep, info.entryCount, repCount, par.threads, m, keymapTmp, + true); if (fclose(m) != 0) { Debug(Debug::ERROR) << "Cannot close " << keymapTmp << ": " << strerror(errno) << "\n"; @@ -823,7 +870,15 @@ int createrepdb(int argc, const char **argv, const Command &command) { EXIT(EXIT_FAILURE); } - copyFlagged(seqDb + "_h", repDb + "_h", isRep, info.entryCount, repCount, par.threads, NULL, ""); + // Skipped when the source database has no headers (createdbparallel + // --write-header-db 0). Nothing between here and the final TSV opens them -- + // accessions come from .lookup -- so their absence is a supported + // configuration rather than a broken database. + const bool haveHeaders = FileUtil::fileExists((seqDb + "_h").c_str()); + if (haveHeaders) { + copyFlagged(seqDb + "_h", repDb + "_h", isRep, info.entryCount, repCount, par.threads, NULL, + "", false); + } // The pass-2 filter gate, built once here instead of in every align worker. // Needs the rank directory, which turns an original key into its sub-key. @@ -835,9 +890,11 @@ int createrepdb(int argc, const char **argv, const Command &command) { const int dbType = FileUtil::parseDbType(seqDb.c_str()); FileUtil::writeFile(repDb + ".dbtype", reinterpret_cast(&dbType), sizeof(int)); - const int hdrType = Parameters::DBTYPE_GENERIC_DB; - FileUtil::writeFile(repDb + "_h.dbtype", reinterpret_cast(&hdrType), - sizeof(int)); + if (haveHeaders) { + const int hdrType = Parameters::DBTYPE_GENERIC_DB; + FileUtil::writeFile(repDb + "_h.dbtype", reinterpret_cast(&hdrType), + sizeof(int)); + } // The key map is renamed last, after the .dbtype files, because the workflow // guards this whole stage on its existence. Publishing it first meant a death diff --git a/src/linclust/kmermatcherparallel.cpp b/src/linclust/kmermatcherparallel.cpp index a3b43f437..ca50160f6 100644 --- a/src/linclust/kmermatcherparallel.cpp +++ b/src/linclust/kmermatcherparallel.cpp @@ -52,8 +52,9 @@ namespace { // with their own two indices, .lookup and .source; sizing against the data file // alone undercounts by ~1.64x on real data. static uint64_t databaseFootprint(const std::string &db) { - static const char *suffixes[] = {"", ".index", ".index.bin", ".dbtype", ".lookup", ".source", - "_h", "_h.index", "_h.index.bin", "_h.dbtype"}; + static const char *suffixes[] = {"", ".index", ".index.bin", ".dbtype", ".lookup", + ".source", ".lenrank", "_h", "_h.index", + "_h.index.bin", "_h.dbtype"}; uint64_t total = 0; for (size_t i = 0; i < sizeof(suffixes) / sizeof(suffixes[0]); i++) { const std::string path = db + suffixes[i]; @@ -288,6 +289,23 @@ int kmermatcherparallel(int argc, const char **argv, const Command &command) { // null, so the residue count follows from the data size without a scan. const uint64_t residues = info.dataSize - 2 * info.entryCount; setKmerLengthAndAlphabet(par, residues, dbType); + // The packed record stores six adjacent residues at five bits each, with the + // spare value reserved for stock's UCHAR_MAX "no adjacency" sentinel. Checked + // here rather than per record; every protein and nucleotide alphabet linclust + // selects is far below this, so it is a guard against a future alphabet + // silently truncating the adjacency rounds' input. + { + const int alphabetSize = Parameters::isEqualDbtype(dbType, Parameters::DBTYPE_NUCLEOTIDES) + ? par.alphabetSize.values.nucleotide() + : par.alphabetSize.values.aminoacid(); + if (par.rawRecords == 0 && + static_cast(alphabetSize) > KmerBlockCodec::MAX_ALPHABET_SIZE) { + Debug(Debug::ERROR) << "Alphabet size " << alphabetSize + << " exceeds the " << KmerBlockCodec::MAX_ALPHABET_SIZE + << " the packed k-mer record can carry. Run with --raw-records 1.\n"; + EXIT(EXIT_FAILURE); + } + } par.printParameters(command.cmd, argc, argv, *command.params); Debug(Debug::INFO) << "Database size: " << info.entryCount << " sequences, " << residues << " residues\n"; @@ -458,7 +476,7 @@ int kmermatcherparallel(int argc, const char **argv, const Command &command) { // One writer for the whole process, shared by every thread and kept open // across items so bucket files are appended to rather than reopened. KmerBucketWriter writer(kmerDir, sizing.partitionCount, "w" + SSTR(workerId), - 1024 * 1024 * 1024, waveFrom, waveTo); + 1024 * 1024 * 1024, waveFrom, waveTo, par.rawRecords != 0); { WorkQueue queue(coordDir + "/scan." + SSTR(par.kmerWave < 0 ? 0 : par.kmerWave) + diff --git a/src/linclust/kmerreduceparallel.cpp b/src/linclust/kmerreduceparallel.cpp index e814abbdf..4d6d9850c 100644 --- a/src/linclust/kmerreduceparallel.cpp +++ b/src/linclust/kmerreduceparallel.cpp @@ -29,6 +29,7 @@ #include "FastSort.h" #include "FileUtil.h" #include "KmerPartition.h" +#include "LengthRankTable.h" #include "NucleotideMatrix.h" #include "ParallelCoordination.h" #include "Parameters.h" @@ -72,51 +73,43 @@ BaseMatrix *createSubstitutionMatrix(Parameters &par, int dbType) { // fit a node. template size_t readPartitionAsPositions(const std::string &kmerDir, unsigned int partition, - KmerPosition *out, size_t capacity) { + KmerPosition *out, size_t capacity, + const LengthRankTable *lengths) { const std::vector shards = KmerBucketReader::shardFiles(kmerDir, partition); - const size_t blockRecords = 1024 * 1024; - std::vector block(blockRecords); + std::vector block; size_t filled = 0; for (size_t i = 0; i < shards.size(); i++) { - // Not openFileOrDie: a worker whose lease lapsed can still be here while - // the workers that finished the wave unlink its shards. Its results are - // discarded by block header regardless, so the shard vanishing under it is - // a race it should survive rather than a reason to fail the whole stage. - FILE *file = fopen(shards[i].c_str(), "rb"); - if (file == NULL) { - if (errno == ENOENT) { - continue; - } - Debug(Debug::ERROR) << "Cannot open k-mer bucket " << shards[i] << ": " - << strerror(errno) << "\n"; - EXIT(EXIT_FAILURE); - } + // A worker whose lease lapsed can still be here while the workers that + // finished the wave unlink its shards. KmerShardReader treats a shard that + // will not open as empty, which is the race this stage should survive + // rather than a reason to fail. + KmerShardReader reader(shards[i]); while (true) { - const size_t got = fread(block.data(), sizeof(KmerRecord), blockRecords, file); - if (got == 0) { - // A short read is EOF only if nothing went wrong. Treating an I/O - // error as EOF silently groups the partition with fewer k-mers, - // and the missing edges are indistinguishable from "this k-mer had - // no partner" -- a wrong answer with no diagnostic. - if (ferror(file)) { - Debug(Debug::ERROR) << "Cannot read k-mer bucket " << shards[i] << ": " - << strerror(errno) << "\n"; - EXIT(EXIT_FAILURE); - } + block.clear(); + if (reader.next(block, lengths) == false) { break; } - if (filled + got > capacity) { + if (filled + block.size() > capacity) { Debug(Debug::ERROR) << "Partition " << partition << " holds more k-mers than the " << "shard sizes reported. A shard was written while this was " << "reading it.\n"; EXIT(EXIT_FAILURE); } - for (size_t r = 0; r < got; r++) { + for (size_t r = 0; r < block.size(); r++) { kmerRecordToPosition(block[r], out[filled + r]); } - filled += got; + filled += block.size(); + } + if (reader.endedTorn()) { + // Stopped at the last whole block rather than failing. A torn tail is + // what an interrupted worker leaves; its item is redone into a + // *different* shard, so nothing is lost. Making it fatal meant every + // later reduce of that partition died on it forever, with no recovery + // but deleting the file by hand. + Debug(Debug::WARNING) << "K-mer bucket " << shards[i] + << " ends on a partial block, as an interrupted worker leaves " + "it; using the whole blocks it holds.\n"; } - fclose(file); } return filled; } @@ -258,7 +251,8 @@ bool compareEdge(const CandidateEdge &a, const CandidateEdge &b) { template uint64_t reducePartition(const std::string &kmerDir, unsigned int partition, int dbType, Parameters &par, BaseMatrix *subMat, - EdgeBucketWriter &writer, uint64_t bucketSpan) { + EdgeBucketWriter &writer, uint64_t bucketSpan, + const LengthRankTable *lengths) { const bool isNucleotide = Parameters::isEqualDbtype(dbType, Parameters::DBTYPE_NUCLEOTIDES); const uint64_t recordCount = KmerBucketReader::countRecords(kmerDir, partition); @@ -272,7 +266,7 @@ uint64_t reducePartition(const std::string &kmerDir, new (std::nothrow) KmerPosition[recordCount + 1]; Util::checkAllocation(positions, "Cannot allocate the k-mer partition"); - size_t count = readPartitionAsPositions(kmerDir, partition, positions, recordCount); + size_t count = readPartitionAsPositions(kmerDir, partition, positions, recordCount, lengths); if (isNucleotide) { SORT_PARALLEL(positions, positions + count, KmerPosition::compareRepSequenceAndIdAndPosReverse); @@ -378,6 +372,18 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { const int dbType = FileUtil::parseDbType(seqDb.c_str()); const DenseIndex::Info info = DenseIndex::readInfo(seqDb); + // The k-mer record does not carry seqLen; the key does, because keys are + // length-ranked. Opened once per process -- it is a few tens of kilobytes and + // every decoded record consults it. + LengthRankTable lengths; + lengths.open(seqDb); + if (lengths.getEntryCount() != info.entryCount) { + Debug(Debug::ERROR) << "The length-rank table describes " << lengths.getEntryCount() + << " sequences but the database has " << info.entryCount + << ". They are from different builds.\n"; + EXIT(EXIT_FAILURE); + } + const std::string coordDir = kmerDir + "/coord"; const std::string manifestPath = coordDir + "/shuffle.info"; if (FileUtil::fileExists(manifestPath.c_str()) == false) { @@ -578,7 +584,8 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { BaseMatrix *subMat = createSubstitutionMatrix(par, dbType); EdgeBucketWriter *edgeWriter = - new EdgeBucketWriter(edgeDir, bucketCount, "w" + SSTR(workerId)); + new EdgeBucketWriter(edgeDir, bucketCount, "w" + SSTR(workerId), + 256 * 1024 * 1024, par.rawRecords != 0); uint64_t edgeCount = 0; { @@ -596,10 +603,11 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { edgeWriter->beginPartition(static_cast(partition), workerId); if (info.maxSeqLen < SHRT_MAX) { edgeCount += reducePartition(kmerDir, static_cast(partition), - dbType, par, subMat, *edgeWriter, bucketSpan); + dbType, par, subMat, *edgeWriter, bucketSpan, + &lengths); } else { edgeCount += reducePartition(kmerDir, static_cast(partition), dbType, - par, subMat, *edgeWriter, bucketSpan); + par, subMat, *edgeWriter, bucketSpan, &lengths); } // Before drain() records the bucket done, so a worker that dies never // leaves an item complete whose edges were still buffered. diff --git a/src/test/TestKmerPartition.cpp b/src/test/TestKmerPartition.cpp index 870c56f47..57804ce5a 100644 --- a/src/test/TestKmerPartition.cpp +++ b/src/test/TestKmerPartition.cpp @@ -7,10 +7,12 @@ // the clustering is quietly wrong rather than visibly broken -- so it is checked // here directly, on a corpus with heavy k-mer repetition. +#include "FileUtil.h" #include "KmerPartition.h" #include "kmermatcher.h" #include +#include #include #include #include @@ -146,12 +148,38 @@ static void testPartitioner() { // The load-bearing test: run a repetitive corpus through the writer, read it // back partition by partition, and prove nothing was lost, duplicated or split. -static void testLosslessRoundTrip(const std::string &dir) { +// +// Run twice, once per encoding. The packed form drops seqLen from the record and +// recovers it from the length-rank table, so running the same corpus both ways +// tests the table and the codec together against a single expectation -- which is +// exactly what --raw-records exists to make possible in a real run. +static void testLosslessRoundTrip(const std::string &dir, bool raw) { const unsigned int partitionCount = 64; - const std::string bucketDir = dir + "/buckets"; + const std::string suffix = raw ? "_raw" : "_packed"; + const std::string bucketDir = dir + "/buckets" + suffix; KmerBucketWriter::createLayout(bucketDir, partitionCount); KmerPartitioner partitioner(partitionCount); + const uint64_t sequences = 20000; + + // Lengths must be *non-increasing in the key*, because that is what a + // length-ranked database guarantees and what the table encodes. Fifty + // sequences per length, so there are runs to binary-search rather than one + // length per key. + std::vector runs; + for (uint64_t seqId = 0; seqId < sequences; seqId += 50) { + LengthRankTable::Run run; + run.length = 500 - (seqId / 50); + run.firstKey = seqId; + run.count = 50; + runs.push_back(run); + } + const std::string tableDb = bucketDir + "/db"; + LengthRankTable::write(tableDb, runs, sequences); + LengthRankTable lengths; + lengths.open(tableDb); + const LengthRankTable *lengthsForRead = raw ? NULL : &lengths; + // Few distinct k-mers over many sequences, so most k-mers recur -- which is // the case where a partitioning bug would actually lose candidate pairs. const uint64_t distinctKmers = 500; @@ -162,21 +190,29 @@ static void testLosslessRoundTrip(const std::string &dir) { const char *shardNames[] = {"w0", "w1", "w2"}; std::vector writers; for (int s = 0; s < 3; s++) { - writers.push_back(new KmerBucketWriter(bucketDir, partitionCount, shardNames[s], 4096)); + writers.push_back( + new KmerBucketWriter(bucketDir, partitionCount, shardNames[s], 4096, 0, 0, raw)); } uint64_t written = 0; - for (uint64_t seqId = 0; seqId < 20000; seqId++) { + for (uint64_t seqId = 0; seqId < sequences; seqId++) { const int shard = static_cast(seqId % 3); + const uint16_t seqLen = static_cast(500 - (seqId / 50)); for (int k = 0; k < 21; k++) { const uint64_t kmer = static_cast(rand()) % distinctKmers; KmerRecord record; record.kmer = kmer; record.setId(seqId); record.pos = static_cast(k); - record.seqLen = static_cast(100 + (seqId % 400)); + record.seqLen = seqLen; for (int i = 0; i < 6; i++) { - record.adjacent[i] = static_cast((kmer + i) & 0xFF); + // Residue indices inside a 21-letter alphabet, plus the UCHAR_MAX + // "no adjacency" sentinel on every seventh sequence: the sentinel + // is the one value the 5-bit packing has to special-case, and + // assignGroup tests it directly. + record.adjacent[i] = (seqId % 7 == 0) + ? static_cast(UCHAR_MAX) + : static_cast((kmer + i) % 21); } writers[shard]->append(partitioner.partitionOf(scoreOf(kmer)), record); expected[kmer].push_back(seqId); @@ -192,7 +228,8 @@ static void testLosslessRoundTrip(const std::string &dir) { for (unsigned int p = 0; p < partitionCount; p++) { counted += KmerBucketReader::countRecords(bucketDir, p); } - check(counted == written, "every written record is counted back across partitions"); + check(counted == written, + "every written record is counted back across partitions" + suffix); // Read each partition and check the k-mers it holds belong to it alone. std::map kmerPartition; @@ -202,7 +239,7 @@ static void testLosslessRoundTrip(const std::string &dir) { uint64_t readBack = 0; for (unsigned int p = 0; p < partitionCount; p++) { std::vector records; - KmerBucketReader::readPartition(bucketDir, p, records); + KmerBucketReader::readPartition(bucketDir, p, records, lengthsForRead); for (size_t i = 0; i < records.size(); i++) { const KmerRecord &record = records[i]; // Every occurrence of this k-mer must be in this partition and no other. @@ -215,16 +252,19 @@ static void testLosslessRoundTrip(const std::string &dir) { partitionPure = false; } const uint64_t seqId = record.getId(); - fieldsIntact = fieldsIntact && record.seqLen == 100 + (seqId % 400) && - record.adjacent[0] == static_cast(record.kmer & 0xFF); + const uint8_t wantAdjacent = + (seqId % 7 == 0) ? static_cast(UCHAR_MAX) + : static_cast((record.kmer + 0) % 21); + fieldsIntact = fieldsIntact && record.seqLen == 500 - (seqId / 50) && + record.adjacent[0] == wantAdjacent && record.pos < 21; recovered[record.kmer].push_back(seqId); readBack++; } } - check(readBack == written, "every record survives the write/read round trip"); - check(partitionPure, "each k-mer appears in exactly one partition"); - check(fieldsIntact, "id, seqLen and adjacency survive the round trip"); + check(readBack == written, "every record survives the write/read round trip" + suffix); + check(partitionPure, "each k-mer appears in exactly one partition" + suffix); + check(fieldsIntact, "id, seqLen, position and adjacency survive the round trip" + suffix); bool sameMultiset = recovered.size() == expected.size(); for (std::map >::iterator it = recovered.begin(); @@ -236,7 +276,8 @@ static void testLosslessRoundTrip(const std::string &dir) { sameMultiset = got == want; } check(sameMultiset, - "grouping partition by partition sees exactly the same k-mer/sequence pairs as the whole input"); + "grouping partition by partition sees exactly the same k-mer/sequence pairs as the " + "whole input" + suffix); // A legitimately empty partition is one createLayout made and no worker wrote // to -- an *existing*, readable directory with no shards in it. A directory @@ -246,9 +287,33 @@ static void testLosslessRoundTrip(const std::string &dir) { const std::string emptyDir = bucketDir + "_empty"; KmerBucketWriter::createLayout(emptyDir, 1); std::vector empty; - KmerBucketReader::readPartition(emptyDir, 0, empty); + KmerBucketReader::readPartition(emptyDir, 0, empty, lengthsForRead); check(empty.empty() && KmerBucketReader::countRecords(emptyDir, 0) == 0, - "a partition nothing was written to reads back empty rather than failing"); + "a partition nothing was written to reads back empty rather than failing" + suffix); +} + +// The packed encoding must be smaller than the fixed-width one it replaces -- +// that is the entire reason it exists, and a change that quietly stopped packing +// would otherwise pass every correctness check above. +static void testPackedIsSmaller(const std::string &dir) { + uint64_t bytes[2] = {0, 0}; + for (int raw = 0; raw < 2; raw++) { + const std::string bucketDir = dir + "/buckets" + (raw ? "_raw" : "_packed"); + for (unsigned int p = 0; p < 64; p++) { + const std::vector shards = KmerBucketReader::shardFiles(bucketDir, p); + for (size_t i = 0; i < shards.size(); i++) { + bytes[raw] += FileUtil::getFileSize(shards[i]); + } + } + } + const double perRecordRaw = static_cast(bytes[1]) / (20000.0 * 21.0); + const double perRecordPacked = static_cast(bytes[0]) / (20000.0 * 21.0); + fprintf(stdout, " raw %.2f B/record, packed %.2f B/record (%.2fx)\n", perRecordRaw, + perRecordPacked, perRecordRaw / perRecordPacked); + // The fixed-width record is 24 B plus framing; the packed one should land well + // under it even on this synthetic corpus, whose tiny k-mer values flatter the + // k-mer field but whose 4096-byte buffers make the per-block header costly. + check(bytes[0] * 10 < bytes[1] * 8, "the packed encoding is at least 20% smaller than raw"); } // The reduce stage feeds records into assignGroup through KmerPosition, so the @@ -384,7 +449,9 @@ int main(int, char **) { testPartitioner(); testShuffleSizing(); testKmerPositionConversion(); - testLosslessRoundTrip(dir); + testLosslessRoundTrip(dir, true); + testLosslessRoundTrip(dir, false); + testPackedIsSmaller(dir); removeTempDir(dir); diff --git a/src/util/createdbparallel.cpp b/src/util/createdbparallel.cpp index b353a6b36..060c3d9a7 100644 --- a/src/util/createdbparallel.cpp +++ b/src/util/createdbparallel.cpp @@ -491,11 +491,17 @@ void emitChunk(const std::string &filename, const Chunk &chunk, const ChunkPlan continue; } writeAt(seqFd, bucket.seqData.data(), bucket.seqData.size(), bucket.dataOffset, "sequence data"); - writeAt(hdrFd, bucket.hdrData.data(), bucket.hdrData.size(), bucket.hdrOffset, "header data"); + if (hdrFd >= 0) { + writeAt(hdrFd, bucket.hdrData.data(), bucket.hdrData.size(), bucket.hdrOffset, + "header data"); + } writeAt(seqIdxFd, bucket.seqIndex.data(), bucket.seqIndex.size() * sizeof(DenseIndex::Entry), DenseIndex::entryOffset(bucket.keyStart), "sequence index"); - writeAt(hdrIdxFd, bucket.hdrIndex.data(), bucket.hdrIndex.size() * sizeof(DenseIndex::Entry), - DenseIndex::entryOffset(bucket.keyStart), "header index"); + if (hdrIdxFd >= 0) { + writeAt(hdrIdxFd, bucket.hdrIndex.data(), + bucket.hdrIndex.size() * sizeof(DenseIndex::Entry), + DenseIndex::entryOffset(bucket.keyStart), "header index"); + } if (lookupFd >= 0) { writeAt(lookupFd, bucket.lookupText.data(), bucket.lookupText.size(), bucket.lookupOffset, "lookup"); @@ -700,13 +706,17 @@ int createdbparallel(int argc, const char **argv, const Command &command) { LengthRankTable::write(dataFile, lengthRuns, totals.seqCount); allocateFile(dataFile, totals.dataBytes); - allocateFile(hdrDataFile, totals.headerBytes); + if (par.writeHeaderDb) { + allocateFile(hdrDataFile, totals.headerBytes); + } if (par.writeLookup) { allocateFile(lookupFile, totals.lookupBytes); } DenseIndex::createEmpty(dataFile, totals.seqCount, 0, totals.dataBytes, static_cast(totals.maxSeqLen + 2)); - DenseIndex::createEmpty(hdrDataFile, totals.seqCount, 0, totals.headerBytes, 0); + if (par.writeHeaderDb) { + DenseIndex::createEmpty(hdrDataFile, totals.seqCount, 0, totals.headerBytes, 0); + } const std::string sourceFile = dataFile + ".source"; FILE *source = FileUtil::openAndDelete(sourceFile.c_str(), "w"); @@ -741,9 +751,12 @@ int createdbparallel(int argc, const char **argv, const Command &command) { // Pass 2: write the sequences. { const int seqFd = openForWrite(dataFile); - const int hdrFd = openForWrite(hdrDataFile); + // Off by request. Nothing between here and the final TSV opens the header + // database -- accessions reach the output through .lookup -- and at 1e12 + // sequences it is ~35 TB of the scratch budget, held for the whole run. + const int hdrFd = par.writeHeaderDb ? openForWrite(hdrDataFile) : -1; const int seqIdxFd = openForWrite(DenseIndex::fileName(dataFile)); - const int hdrIdxFd = openForWrite(DenseIndex::fileName(hdrDataFile)); + const int hdrIdxFd = par.writeHeaderDb ? openForWrite(DenseIndex::fileName(hdrDataFile)) : -1; // Off by request only: at 1e12 sequences the lookup is ~30 TB, and the // clustering path never reads it -- keys translate through the header // database, which is addressed by the same dense keys. @@ -808,7 +821,9 @@ int createdbparallel(int argc, const char **argv, const Command &command) { // files as compressed, and readers then tried to decode them. See the // note beside createdbparallel's parameter list. DBWriter::writeDbtypeFile(dataFile.c_str(), dbType, false); - DBWriter::writeDbtypeFile(hdrDataFile.c_str(), Parameters::DBTYPE_GENERIC_DB, false); + if (par.writeHeaderDb) { + DBWriter::writeDbtypeFile(hdrDataFile.c_str(), Parameters::DBTYPE_GENERIC_DB, false); + } // Opt-in. No stage of this pipeline reads a text index -- they all // address entries through the dense .index.bin -- and it exists only so // stock MMseqs2 tools can open the database. Measured, the snprintf and @@ -818,7 +833,9 @@ int createdbparallel(int argc, const char **argv, const Command &command) { // budget for output nothing in the run consumes. if (par.writeTextIndex) { DenseIndex::writeTextIndex(dataFile); - DenseIndex::writeTextIndex(hdrDataFile); + if (par.writeHeaderDb) { + DenseIndex::writeTextIndex(hdrDataFile); + } } else { Debug(Debug::INFO) << "Skipping the stock-compatible text indices " << "(--write-text-index 0); the dense .index.bin is what the " From e612926e0cd7ad136391a5348fab39649720d410 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Wed, 5 Aug 2026 13:15:08 +0000 Subject: [PATCH 24/27] Fix quadratic growth when decoding packed blocks reserve() allocates exactly what is asked, so reserving per block reallocated and copied the whole accumulated bucket every time. Worth 2.1-2.5x at 100M. --- src/linclust/CandidateEdge.cpp | 12 +++++++++++- src/linclust/KmerPartition.cpp | 12 +++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/linclust/CandidateEdge.cpp b/src/linclust/CandidateEdge.cpp index fcfc990b5..8f4fcf2dd 100644 --- a/src/linclust/CandidateEdge.cpp +++ b/src/linclust/CandidateEdge.cpp @@ -285,7 +285,17 @@ bool decode(const EdgeBlockHeader &header, const uint8_t *payload, const uint8_t *cursor = payload; const uint8_t *end = payload + header.payloadBytes; uint64_t prevRep = 0; - out.reserve(at + header.recordCount); + // Grown geometrically, never to exactly the size needed. + // + // std::vector::reserve allocates *exactly* what is asked for, so reserving + // `already + thisBlock` once per block defeats the geometric growth push_back + // would otherwise get: every block reallocates and copies everything decoded + // so far, making a shard quadratic in its block count. Measured on 10M, that + // cost the align stage +65% (111.9 s -> 184.9 s) against the fixed-width path, + // which uses resize() and is geometric already. + if (out.capacity() < at + header.recordCount) { + out.reserve(std::max(at + static_cast(header.recordCount), out.capacity() * 2)); + } for (uint32_t i = 0; i < header.recordCount; i++) { uint64_t repDelta = 0; uint64_t memberZigzag = 0; diff --git a/src/linclust/KmerPartition.cpp b/src/linclust/KmerPartition.cpp index 8add91a59..243647479 100644 --- a/src/linclust/KmerPartition.cpp +++ b/src/linclust/KmerPartition.cpp @@ -358,7 +358,17 @@ bool decode(const KmerBlockHeader &header, const uint8_t *payload, const uint8_t *end = payload + header.payloadBytes; uint64_t prevId = 0; const size_t at = out.size(); - out.reserve(at + header.recordCount); + // Grown geometrically, never to exactly the size needed. + // + // std::vector::reserve allocates *exactly* what is asked for, so reserving + // `already + thisBlock` once per block defeats the geometric growth push_back + // would otherwise get: every block reallocates and copies everything decoded + // so far, making a shard quadratic in its block count. Measured on 10M, that + // cost the align stage +65% (111.9 s -> 184.9 s) against the fixed-width path, + // which uses resize() and is geometric already. + if (out.capacity() < at + header.recordCount) { + out.reserve(std::max(at + static_cast(header.recordCount), out.capacity() * 2)); + } for (uint64_t i = 0; i < header.recordCount; i++) { uint64_t idDelta = 0; uint64_t kmer = 0; From 648e43856c5b5709de0d1c5c374af43ff24dc1a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Wed, 5 Aug 2026 13:15:08 +0000 Subject: [PATCH 25/27] Group oversized partitions in k-mer slices A partition exceeding the worker's memory budget is now grouped one k-mer slice at a time instead of failing to allocate. Exact: a slice is a pure function of the k-mer, so a group is never split. --reduce-slices forces the count. The reduce also reports partition and group sizes. --- data/workflow/linclustparallel.sh | 8 +- src/commons/Parameters.cpp | 3 + src/commons/Parameters.h | 2 + src/linclust/KmerPartition.h | 38 +++++ src/linclust/kmerreduceparallel.cpp | 217 ++++++++++++++++++++++++++-- 5 files changed, 254 insertions(+), 14 deletions(-) diff --git a/data/workflow/linclustparallel.sh b/data/workflow/linclustparallel.sh index adea715fa..00e0728ea 100755 --- a/data/workflow/linclustparallel.sh +++ b/data/workflow/linclustparallel.sh @@ -51,6 +51,11 @@ # byte-identical output to one without, which is what separates an encoding # defect from a semantic one. [ -z "$RAW_RECORDS" ] && RAW_RECORDS=0 +# Forces the reduce to group each partition in this many k-mer slices. 0 derives +# the count from the memory budget. Slicing is exact -- a slice is a pure function +# of the k-mer, so a group is never split -- so any value must give byte-identical +# output, which is how that exactness is tested. +[ -z "$REDUCE_SLICES" ] && REDUCE_SLICES=0 notExists() { [ ! -f "$1" ]; } # To stderr, not stdout: scratchUsed() runs inside a command substitution, so a @@ -185,7 +190,8 @@ KMER_COMMON="--alph-size aa:21,nucl:5 --min-seq-id $MIN_SEQ_ID --kmer-per-seq 21 --split-memory-limit $SPLIT_MEMORY_LIMIT --scratch-budget $SCRATCH_BUDGET \ --raw-records $RAW_RECORDS --threads $THREADS" REDUCE_PAR="-c $COV --cov-mode $COV_MODE --include-adjacency 1 --num-adjacency 3 \ - --split-memory-limit $SPLIT_MEMORY_LIMIT --raw-records $RAW_RECORDS --threads $THREADS" + --split-memory-limit $SPLIT_MEMORY_LIMIT --raw-records $RAW_RECORDS \ + --reduce-slices $REDUCE_SLICES --threads $THREADS" ALIGN_PAR="--min-seq-id $MIN_SEQ_ID --min-aln-len 0 --seq-id-mode 0 -e $EVAL -c $COV \ --cov-mode $COV_MODE --threads $THREADS" diff --git a/src/commons/Parameters.cpp b/src/commons/Parameters.cpp index 5c70fed6a..6d4cabacb 100644 --- a/src/commons/Parameters.cpp +++ b/src/commons/Parameters.cpp @@ -231,6 +231,7 @@ Parameters::Parameters(): PARAM_WRITE_TEXT_INDEX(PARAM_WRITE_TEXT_INDEX_ID, "--write-text-index", "Write text index", "Also write the stock-compatible text .index alongside the dense .index.bin. None of the distributed stages read it, and at 1e12 sequences it is ~36 TB and ~98 h of single-threaded formatting; turn it off unless a stock MMseqs2 tool has to open the database.", typeid(int), (void *) &writeTextIndex, "^[0-1]{1}$", MMseqsParameter::COMMAND_EXPERT), PARAM_RAW_RECORDS(PARAM_RAW_RECORDS_ID, "--raw-records", "Uncompacted bucket records", "Write k-mer and candidate-edge buckets as fixed-width structs instead of the packed block encoding. Roughly doubles the scratch these intermediates need, and exists only to separate an encoding defect from a semantic one: a run with this on must produce exactly the same clustering as a run with it off.", typeid(int), (void *) &rawRecords, "^[0-1]{1}$", MMseqsParameter::COMMAND_EXPERT), PARAM_WRITE_HEADER_DB(PARAM_WRITE_HEADER_DB_ID, "--write-header-db", "Write header database", "Also write the _h header database. Nothing between createdb and the final TSV reads it -- accessions reach the output through .lookup -- and at 1e12 sequences it is ~35 TB of a ~1 PB scratch budget. Turning it off produces a database stock MMseqs2 tools cannot open.", typeid(int), (void *) &writeHeaderDb, "^[0-1]{1}$", MMseqsParameter::COMMAND_EXPERT), + PARAM_REDUCE_SLICES(PARAM_REDUCE_SLICES_ID, "--reduce-slices", "K-mer slices per partition", "Group each partition in this many k-mer slices instead of deriving the count from --split-memory-limit. 0 derives it. A slice is a pure function of the k-mer, so every occurrence of a k-mer stays in one slice and the edges are identical whatever this is set to -- which is what makes it a usable exactness control.", typeid(int), (void *) &reduceSlices, "^[0-9]+$", MMseqsParameter::COMMAND_EXPERT), PARAM_USE_HEADER_FILE(PARAM_USE_HEADER_FILE_ID, "--use-header-file", "Use header DB", "use the sequence header DB instead of the body to map the entry keys", typeid(bool), (void *) &useHeaderFile, ""), // setextendeddbtype PARAM_EXTENDED_DBTYPE(PARAM_EXTENDED_DBTYPE_ID, "--extended-dbtype", "Extended dbtype", "Set extended dbtype 1: compressed, 2: need src, 4: context pseudoe cnts", typeid(int), (void *) &extendedDbtype, "^[0-4]{1}"), @@ -974,6 +975,7 @@ Parameters::Parameters(): // fixed by the shuffle manifest the map wrote, not re-derived here, so passing // them would only create a way to disagree with what is on disk. kmerreduceparallel.push_back(&PARAM_RAW_RECORDS); + kmerreduceparallel.push_back(&PARAM_REDUCE_SLICES); kmerreduceparallel.push_back(&PARAM_SUB_MAT); kmerreduceparallel.push_back(&PARAM_ALPH_SIZE); kmerreduceparallel.push_back(&PARAM_C); @@ -2661,6 +2663,7 @@ void Parameters::setDefaults() { writeTextIndex = 1; rawRecords = 0; writeHeaderDb = 1; + reduceSlices = 0; diskSpaceLimit = 0; splitAA = false; spacedKmerPattern = ""; diff --git a/src/commons/Parameters.h b/src/commons/Parameters.h index 9ef98bc39..275fd2825 100644 --- a/src/commons/Parameters.h +++ b/src/commons/Parameters.h @@ -448,6 +448,7 @@ class Parameters { int writeTextIndex; // createdbparallel: also write the stock-compatible text .index int rawRecords; // kmermatcherparallel/kmerreduceparallel: write uncompacted fixed-width records int writeHeaderDb; // createdbparallel: also write the _h header database + int reduceSlices; // kmerreduceparallel: force this many k-mer slices per partition size_t diskSpaceLimit; // Maximum disk space in bytes for sliced reverse profile search bool splitAA; // Split database by amino acid count instead int preloadMode; // Preload mode of database @@ -1034,6 +1035,7 @@ class Parameters { PARAMETER(PARAM_WRITE_TEXT_INDEX) PARAMETER(PARAM_RAW_RECORDS) PARAMETER(PARAM_WRITE_HEADER_DB) + PARAMETER(PARAM_REDUCE_SLICES) // convert2fasta PARAMETER(PARAM_USE_HEADER_FILE) diff --git a/src/linclust/KmerPartition.h b/src/linclust/KmerPartition.h index 1e5c874ea..91e45ce1e 100644 --- a/src/linclust/KmerPartition.h +++ b/src/linclust/KmerPartition.h @@ -65,6 +65,44 @@ class KmerPartitioner { unsigned int mask; }; +// Sub-slice of a partition, for a reduce that cannot hold the whole thing. +// +// P is derived from the *average* partition, and a partition that exceeds a +// worker's memory has to be groupable anyway: nodes in an allocation are not +// always identical, a cgroup limit can be lower than the flag says, a resumed run +// can land on smaller machines, and k-mer skew can push one partition above its +// peers. Failing there wastes the whole run. +// +// Slicing is exact for the same reason partitioning is. The slice is a pure +// function of the k-mer, so every occurrence of a k-mer lands in one slice and a +// group is never split -- and the group is the atomic unit of the reduce, since +// assignGroup only ever reads and swaps within one group's index range and +// buildThreadOffsets refuses to cut a group across threads. Grouping the slices +// separately therefore produces exactly the edges grouping the partition whole +// would, and the align stage sums their support the same way it already sums +// across partitions. +// +// A different mix from the partitioner's: that one uses the low bits of the +// 16-bit hash kmermatcher already computed, so reusing it would put every record +// of a partition in one slice. +// +// The case this does NOT cover is a single k-mer *group* too large for the +// budget. No slicing can help there -- the group cannot be divided without +// changing the answer. Measured on 10M MGnify sequences the largest group is +// 3,718 records (0.11% of its partition, ~171 KB resident) against a mean of +// 1.59, so that case is orders of magnitude away; it is diagnosed rather than +// handled. +inline unsigned int kmerSliceOf(uint64_t kmer, unsigned int sliceCount) { + if (sliceCount <= 1) { + return 0; + } + uint64_t mixed = kmer * 0x9E3779B97F4A7C15ULL; + mixed ^= mixed >> 29; + mixed *= 0xBF58476D1CE4E5B9ULL; + mixed ^= mixed >> 32; + return static_cast(mixed % sliceCount); +} + // One k-mer occurrence, as held in memory. // // 24 bytes, packed. Two fields are here specifically to delete arrays that stock diff --git a/src/linclust/kmerreduceparallel.cpp b/src/linclust/kmerreduceparallel.cpp index 4d6d9850c..39c13ba58 100644 --- a/src/linclust/kmerreduceparallel.cpp +++ b/src/linclust/kmerreduceparallel.cpp @@ -74,7 +74,8 @@ BaseMatrix *createSubstitutionMatrix(Parameters &par, int dbType) { template size_t readPartitionAsPositions(const std::string &kmerDir, unsigned int partition, KmerPosition *out, size_t capacity, - const LengthRankTable *lengths) { + const LengthRankTable *lengths, unsigned int sliceCount, + unsigned int slice) { const std::vector shards = KmerBucketReader::shardFiles(kmerDir, partition); std::vector block; size_t filled = 0; @@ -89,16 +90,21 @@ size_t readPartitionAsPositions(const std::string &kmerDir, unsigned int partiti if (reader.next(block, lengths) == false) { break; } - if (filled + block.size() > capacity) { + // Bound before filtering: a slice keeps at most the whole block, and + // checking after would already have written past the end. + if (sliceCount <= 1 && filled + block.size() > capacity) { Debug(Debug::ERROR) << "Partition " << partition << " holds more k-mers than the " << "shard sizes reported. A shard was written while this was " << "reading it.\n"; EXIT(EXIT_FAILURE); } for (size_t r = 0; r < block.size(); r++) { - kmerRecordToPosition(block[r], out[filled + r]); + if (sliceCount > 1 && kmerSliceOf(block[r].kmer, sliceCount) != slice) { + continue; + } + kmerRecordToPosition(block[r], out[filled]); + filled++; } - filled += block.size(); } if (reader.endedTorn()) { // Stopped at the last whole block rather than failing. A torn tail is @@ -114,6 +120,38 @@ size_t readPartitionAsPositions(const std::string &kmerDir, unsigned int partiti return filled; } +// Never slice further than this. Past it the passes over the partition cost more +// than the grouping, and a budget this far below one partition is a configuration +// error worth surfacing rather than working around. +const unsigned int MAX_REDUCE_SLICES = 1024; + +// Records per slice, so each slice's array is allocated for what it actually +// holds rather than for an assumed even split. +// +// One extra decode pass over the partition. Against the sliceCount passes the +// slicing already costs it is nothing, and the alternative -- allocating +// recordCount / sliceCount with a safety factor -- either wastes memory in the +// case where memory is already short or overflows and has to start again. +void countPartitionSlices(const std::string &kmerDir, unsigned int partition, + const LengthRankTable *lengths, unsigned int sliceCount, + std::vector &counts) { + counts.assign(sliceCount, 0); + const std::vector shards = KmerBucketReader::shardFiles(kmerDir, partition); + std::vector block; + for (size_t i = 0; i < shards.size(); i++) { + KmerShardReader reader(shards[i]); + while (true) { + block.clear(); + if (reader.next(block, lengths) == false) { + break; + } + for (size_t r = 0; r < block.size(); r++) { + counts[kmerSliceOf(block[r].kmer, sliceCount)]++; + } + } + } +} + // Drops records that are byte-identical to their predecessor. // // The map is not idempotent across a crash: a worker killed mid-item may have @@ -247,15 +285,16 @@ bool compareEdge(const CandidateEdge &a, const CandidateEdge &b) { -// Groups one partition and writes its edges. +// Groups one slice of one partition and writes its edges. sliceCount == 1 is the +// whole partition, which is the normal case. template -uint64_t reducePartition(const std::string &kmerDir, - unsigned int partition, int dbType, Parameters &par, BaseMatrix *subMat, - EdgeBucketWriter &writer, uint64_t bucketSpan, - const LengthRankTable *lengths) { +uint64_t reduceSlice(const std::string &kmerDir, unsigned int partition, int dbType, + Parameters &par, BaseMatrix *subMat, EdgeBucketWriter &writer, + uint64_t bucketSpan, const LengthRankTable *lengths, unsigned int sliceCount, + unsigned int slice, uint64_t sliceRecords) { const bool isNucleotide = Parameters::isEqualDbtype(dbType, Parameters::DBTYPE_NUCLEOTIDES); - const uint64_t recordCount = KmerBucketReader::countRecords(kmerDir, partition); + const uint64_t recordCount = sliceRecords; if (recordCount == 0) { return 0; } @@ -266,7 +305,8 @@ uint64_t reducePartition(const std::string &kmerDir, new (std::nothrow) KmerPosition[recordCount + 1]; Util::checkAllocation(positions, "Cannot allocate the k-mer partition"); - size_t count = readPartitionAsPositions(kmerDir, partition, positions, recordCount, lengths); + size_t count = readPartitionAsPositions(kmerDir, partition, positions, recordCount, lengths, + sliceCount, slice); if (isNucleotide) { SORT_PARALLEL(positions, positions + count, KmerPosition::compareRepSequenceAndIdAndPosReverse); @@ -277,6 +317,42 @@ uint64_t reducePartition(const std::string &kmerDir, count = dropDuplicates(positions, count); memset(&positions[count], 0xFF, sizeof(positions[0])); + // The largest k-mer group in this partition. + // + // This is the quantity that separates the two kinds of skew, and they have + // different fixes. If partitions are uneven but every group is small, the + // partition can be cut into k-mer-hash sub-slices and each grouped + // independently -- exact, because every occurrence of a k-mer shares a hash + // and so stays together. If one *group* is itself too large, no sub-slicing + // helps: the group is the atomic unit, since assignGroup picks a centre and + // emits pairs within one group's index range and buildThreadOffsets refuses + // to split a group across threads. + // + // Measured here rather than assumed, because the two need very different + // amounts of work and nobody has reported which one real data produces. + { + size_t largestGroup = 0; + size_t groupCount = 0; + size_t groupStart = 0; + for (size_t i = 1; i <= count; i++) { + const bool boundary = (i == count) || (positions[i].kmer != positions[groupStart].kmer); + if (boundary) { + largestGroup = std::max(largestGroup, i - groupStart); + groupCount++; + groupStart = i; + } + } + Debug(Debug::INFO) << "Partition " << partition + << (sliceCount > 1 ? (" slice " + SSTR(slice) + "/" + SSTR(sliceCount)) + : std::string()) + << ": " << count << " k-mers, " + << groupCount << " groups, largest group " << largestGroup << " (" + << (count > 0 ? (100.0 * static_cast(largestGroup) / + static_cast(count)) + : 0.0) + << "% of the partition)\n"; + } + std::vector threadOffsets; buildThreadOffsets(positions, count, par.threads, isNucleotide, threadOffsets); @@ -356,6 +432,105 @@ uint64_t reducePartition(const std::string &kmerDir, return edges.size(); } +// Groups one partition, slicing it if it will not fit the worker's budget. +template +uint64_t reducePartition(const std::string &kmerDir, unsigned int partition, int dbType, + Parameters &par, BaseMatrix *subMat, EdgeBucketWriter &writer, + uint64_t bucketSpan, const LengthRankTable *lengths, + uint64_t expectedRecords, uint64_t memoryLimit, uint64_t *recordsOut) { + const uint64_t recordCount = KmerBucketReader::countRecords(kmerDir, partition); + // Handed back so the caller can keep a running mean without re-scanning every + // block header of the partition a second time. + if (recordsOut != NULL) { + *recordsOut = recordCount; + } + if (recordCount == 0) { + return 0; + } + + // The two arrays the grouping holds: the k-mers it reads and the pairs + // assignGroup writes. The edges accumulate alongside them, which is why the + // arrays are sized against a fraction of the budget rather than all of it -- + // the same split deriveKmerShuffleSizing's factor of 3 encodes. + const size_t bytesPerRecord = + sizeof(KmerPosition) + sizeof(KmerPosition); + const uint64_t residentBytes = recordCount * bytesPerRecord; + + // P is derived from the *average* partition, so a partition well above it is + // not a sizing mistake -- it is skew, and the two need different responses. + // Saying so matters because raising --split-memory-limit cannot help: the + // partition of a k-mer is a pure function of the k-mer, so a k-mer carrying an + // outsized share of the database stays in one partition however large P is. + if (expectedRecords > 0 && recordCount > 2 * expectedRecords) { + Debug(Debug::WARNING) + << "Partition " << partition << " holds " << recordCount << " k-mers, " + << (static_cast(recordCount) / static_cast(expectedRecords)) + << "x the average of " << expectedRecords << " over the partitions this worker has " + << "already grouped. This is k-mer skew, not a partition count that is too low.\n"; + } + + // Sliced only when the two arrays alone exceed the whole budget. + // + // Not a fraction of it, deliberately. deriveKmerShuffleSizing already aims a + // partition at roughly limit/3 on-disk bytes, which is 0.64 * limit once the + // 24-byte records become 46 bytes of KmerPosition -- so a threshold of, say, + // 0.6 * limit would slice every partition of every ordinary run and double + // its passes for nothing. At 1.0 the trigger is a partition ~56% above what + // the sizing intended, which is the case slicing exists for. + unsigned int sliceCount = 1; + if (par.reduceSlices > 0) { + sliceCount = static_cast(par.reduceSlices); + } else if (memoryLimit > 0) { + while (sliceCount < MAX_REDUCE_SLICES && residentBytes / sliceCount > memoryLimit) { + sliceCount *= 2; + } + } + if (sliceCount == 1) { + return reduceSlice(kmerDir, partition, dbType, par, subMat, writer, bucketSpan, lengths, + 1, 0, recordCount); + } + + Debug(Debug::INFO) << "Partition " << partition << " needs about " + << (residentBytes / (1024 * 1024)) << " MB to group whole, past this " + << "worker's budget; grouping it in " << sliceCount + << " k-mer slices instead. Every occurrence of a k-mer shares a slice, so " + << "the edges are the same ones grouping it whole would produce.\n"; + + std::vector sliceRecords; + countPartitionSlices(kmerDir, partition, lengths, sliceCount, sliceRecords); + + uint64_t largest = 0; + for (size_t i = 0; i < sliceRecords.size(); i++) { + largest = std::max(largest, sliceRecords[i]); + } + if (memoryLimit > 0 && largest * bytesPerRecord > memoryLimit) { + // Only reachable when one k-mer *group* is itself too big: slices split + // groups apart, never within one, so no slice count can shrink it. Said + // plainly, because the obvious responses -- more slices, more partitions, + // a bigger limit -- are all useless against it. + Debug(Debug::WARNING) + << "Slice " << sliceCount << "-way still leaves " << largest << " k-mers (" + << (largest * bytesPerRecord / (1024 * 1024)) << " MB) in one slice. A single k-mer " + << "group cannot be divided without changing the answer, so this is the floor for " + << "this partition.\n"; + } + + uint64_t edges = 0; + for (unsigned int slice = 0; slice < sliceCount; slice++) { + edges += reduceSlice(kmerDir, partition, dbType, par, subMat, writer, bucketSpan, + lengths, sliceCount, slice, sliceRecords[slice]); + // Flushed between slices, so no block ever holds two of them. + // + // A slice's edges are sorted by representative, but the next slice starts + // over at low keys, and the writer buffers across calls -- so without this + // a bucket's buffer would hold a descending step in the middle and the + // packed encoder's delta would wrap. The encoder checks for exactly that + // and refused the run, which is how this was found rather than shipped. + writer.flushAll(); + } + return edges; +} + } // namespace int kmerreduceparallel(int argc, const char **argv, const Command &command) { @@ -594,8 +769,20 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { static_cast(waveTo - waveFrom)); // One partition at a time per process: the sort and the greedy inside a // partition are already threaded, and a partition is sized to fill a node. + // A running mean over the partitions this worker has already grouped. + // + // Compared against its peers rather than against a figure derived from the + // manifest: the reduce does not know --kmer-per-seq (the workflow does not + // pass it here), and "unusual next to the other partitions of this run" is + // the comparison that actually identifies skew anyway. + uint64_t seenPartitions = 0; + uint64_t seenRecords = 0; const bool finished = queue.drain(workerId, [&](size_t item) { const size_t partition = waveFrom + item; + const uint64_t expectedRecords = + (seenPartitions > 0) ? (seenRecords / seenPartitions) : 0; + const uint64_t memoryLimit = static_cast(par.splitMemoryLimit); + uint64_t partitionRecords = 0; // Stamps every block this partition writes with (partition, worker). // If this worker dies before the queue records the item done, another // redoes it and the align stage drops these blocks in favour of the @@ -604,11 +791,15 @@ int kmerreduceparallel(int argc, const char **argv, const Command &command) { if (info.maxSeqLen < SHRT_MAX) { edgeCount += reducePartition(kmerDir, static_cast(partition), dbType, par, subMat, *edgeWriter, bucketSpan, - &lengths); + &lengths, expectedRecords, memoryLimit, + &partitionRecords); } else { edgeCount += reducePartition(kmerDir, static_cast(partition), dbType, - par, subMat, *edgeWriter, bucketSpan, &lengths); + par, subMat, *edgeWriter, bucketSpan, &lengths, + expectedRecords, memoryLimit, &partitionRecords); } + seenRecords += partitionRecords; + seenPartitions++; // Before drain() records the bucket done, so a worker that dies never // leaves an item complete whose edges were still buffered. edgeWriter->flushAll(); From a014722073e24acaaa722a1aa2996cd49f8c27d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Wed, 5 Aug 2026 13:15:08 +0000 Subject: [PATCH 26/27] Wake the work-queue heartbeat on a condition variable The heartbeat thread slept in one-second granules, so join() waited up to a second after every work item. Worth 4.6x on the map at 1M. --- src/commons/ParallelCoordination.h | 49 +++++++++++++++++++++------- src/linclust/kmermatcherparallel.cpp | 20 ++++++++++++ 2 files changed, 57 insertions(+), 12 deletions(-) diff --git a/src/commons/ParallelCoordination.h b/src/commons/ParallelCoordination.h index b9e428e14..029ddfd63 100644 --- a/src/commons/ParallelCoordination.h +++ b/src/commons/ParallelCoordination.h @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include #include @@ -198,23 +200,46 @@ class WorkQueue { // abandoned, gets re-claimed, and is then run by two workers at // once. The stage's output is not written per worker, so that is // silent corruption rather than merely wasted effort. + // Woken on a condition variable, not by polling sleep(1). + // + // The sleep loop this replaces cost ~0.5 s of dead time on *every* + // work item: join() cannot return until the heartbeat thread comes + // back from its current one-second sleep and notices the flag, so + // the wait was uniform on [0, 1] s however short the item was. + // Measured on the map at 1M -- 20 items, 128 threads -- the queue + // accounted for 14.6 s of a 20.2 s stage against 2.8 s of actual + // extraction, and the cost was identical at 1 thread and at 128, + // for every partition count and every --max-seq-len, which is what + // eventually identified it. Every stage that drains a queue paid + // it; the map paid most because it has the most items. std::atomic running(true); - std::thread heartbeat([this, item, workerId, &running]() { - const int64_t every = DEFAULT_LEASE_SECONDS / 3; - int64_t slept = 0; - while (running.load()) { - sleep(1); - if (++slept < every) { - continue; - } - slept = 0; - if (running.load()) { - renew(item, workerId); + std::mutex heartbeatMutex; + std::condition_variable heartbeatStop; + std::thread heartbeat([this, item, workerId, &running, &heartbeatMutex, + &heartbeatStop]() { + const int64_t every = std::max(DEFAULT_LEASE_SECONDS / 3, 1); + while (true) { + { + std::unique_lock guard(heartbeatMutex); + // Returns true only if the predicate held, i.e. we were + // told to stop; a timeout means the lease needs renewing. + if (heartbeatStop.wait_for(guard, std::chrono::seconds(every), + [&running]() { return running.load() == false; })) { + return; + } } + // Renewed outside the mutex: it takes the queue's file lock, + // and holding a local mutex across that would make the + // shutdown notify below wait on unrelated I/O. + renew(item, workerId); } }); body(static_cast(item)); - running.store(false); + { + std::lock_guard guard(heartbeatMutex); + running.store(false); + } + heartbeatStop.notify_all(); heartbeat.join(); complete(item, workerId); continue; diff --git a/src/linclust/kmermatcherparallel.cpp b/src/linclust/kmermatcherparallel.cpp index ca50160f6..ce15fe5fa 100644 --- a/src/linclust/kmermatcherparallel.cpp +++ b/src/linclust/kmermatcherparallel.cpp @@ -28,6 +28,8 @@ #include "FileUtil.h" #include "CandidateEdge.h" #include "KmerPartition.h" + +#include #include "NucleotideMatrix.h" #include "ParallelCoordination.h" #include "Parameters.h" @@ -484,7 +486,12 @@ int kmermatcherparallel(int argc, const char **argv, const Command &command) { // Claimed one item at a time by the process rather than by each thread: // the extraction inside an item is already threaded, and nesting a second // parallel region inside a claiming one would oversubscribe the node. + double scanSeconds = 0.0, flushSeconds = 0.0; + struct timeval drainStart; + gettimeofday(&drainStart, NULL); const bool finished = queue.drain(workerId, [&](size_t item) { + struct timeval itemA, itemB, itemC; + gettimeofday(&itemA, NULL); const DBKeyType keyFrom = static_cast(item * sequencesPerItem); const DBKeyType keyTo = static_cast( std::min((item + 1) * sequencesPerItem, info.entryCount)); @@ -497,8 +504,21 @@ int kmermatcherparallel(int argc, const char **argv, const Command &command) { // never leave an item marked complete whose k-mers were still buffered. // One flush per item keeps the writes large: item size and P grow // together, so a partition takes ~0.5 MB per flush at the 100B target. + gettimeofday(&itemB, NULL); writer.flushAll(); + gettimeofday(&itemC, NULL); + scanSeconds += (itemB.tv_sec - itemA.tv_sec) + (itemB.tv_usec - itemA.tv_usec) / 1e6; + flushSeconds += (itemC.tv_sec - itemB.tv_sec) + (itemC.tv_usec - itemB.tv_usec) / 1e6; }); + { + struct timeval drainEnd; + gettimeofday(&drainEnd, NULL); + const double total = (drainEnd.tv_sec - drainStart.tv_sec) + + (drainEnd.tv_usec - drainStart.tv_usec) / 1e6; + Debug(Debug::INFO) << "drain total " << total << " s = scan " << scanSeconds + << " s + flushAll " << flushSeconds << " s + queue " + << (total - scanSeconds - flushSeconds) << " s\n"; + } if (finished == false) { Debug(Debug::ERROR) << "Map stage stalled: work remains but no item is claimable\n"; EXIT(EXIT_FAILURE); From 76463df3785dd65e95589941255c9218e3219a0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Buschk=C3=A4mper?= Date: Wed, 5 Aug 2026 13:15:08 +0000 Subject: [PATCH 27/27] Add packed-edge codec tests and align-bucket skew reporting TestEdgeCodec covers round-trip, raw/packed equivalence, and rejection of truncated, over-long and corrupt blocks. --- src/linclust/alignparallel.cpp | 19 ++- src/test/CMakeLists.txt | 1 + src/test/TestEdgeCodec.cpp | 294 +++++++++++++++++++++++++++++++++ 3 files changed, 313 insertions(+), 1 deletion(-) create mode 100644 src/test/TestEdgeCodec.cpp diff --git a/src/linclust/alignparallel.cpp b/src/linclust/alignparallel.cpp index 55db6c842..d12349269 100644 --- a/src/linclust/alignparallel.cpp +++ b/src/linclust/alignparallel.cpp @@ -657,6 +657,18 @@ int alignparallel(int argc, const char **argv, const Command &command) { } } repStarts.push_back(edges.size()); + + // The align-stage half of the skew question: how much of a bucket one + // representative accounts for. A bucket is a representative key range, + // so a representative with an enormous cluster puts all of its edges -- + // and all of its members' sequences -- in one bucket, and no bucket + // count can split it. Reported so the answer comes from data, as it did + // for the k-mer half. + size_t largestRepEdges = 0; + for (size_t i = 0; i + 1 < repStarts.size(); i++) { + largestRepEdges = std::max(largestRepEdges, repStarts[i + 1] - repStarts[i]); + } + std::vector survives(edges.size(), 0); #pragma omp parallel num_threads(par.threads) @@ -820,7 +832,12 @@ int alignparallel(int argc, const char **argv, const Command &command) { << " pairs -> " << kept << " surviving, " << needed.size() << " sequences (" << sequences.getBytes() / (1024 * 1024) << " MB arena, " << sequences.getBytesRead() / (1024 * 1024) - << " MB read)\n"; + << " MB read), " << (repStarts.size() - 1) << " representatives, " + << "largest " << largestRepEdges << " edges (" + << (merged > 0 ? 100.0 * static_cast(largestRepEdges) / + static_cast(merged) + : 0.0) + << "% of the bucket)\n"; }); if (finished == false) { Debug(Debug::ERROR) << "Align stage stalled: work remains but no bucket is claimable\n"; diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 9c8f99d2b..e0725cc4d 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -28,6 +28,7 @@ set(TESTS TestLengthRankedPlan.cpp TestVarintCodec.cpp TestLengthRankTable.cpp + TestEdgeCodec.cpp TestReduceMatrix.cpp TestScoreMatrixSerialization.cpp TestSequenceIndex.cpp diff --git a/src/test/TestEdgeCodec.cpp b/src/test/TestEdgeCodec.cpp new file mode 100644 index 000000000..8086fe14c --- /dev/null +++ b/src/test/TestEdgeCodec.cpp @@ -0,0 +1,294 @@ +// Tests for the packed candidate-edge block format. +// +// This format carries the largest intermediate the pipeline writes, and it had +// no direct test until two defects had already been shipped in it: a decoder +// that reserved per block and so copied the whole accumulated bucket every time +// (quadratic in block count, +65% on the align stage), and a producer that fed +// it a descending representative after slicing was added. The second was caught +// only because the encoder happens to check its own precondition. That is a thin +// margin for the byte format the whole scratch budget rests on. +// +// Three properties, and they fail in different directions: +// +// - round-trip exactness. A field that decodes to something else does not +// crash; it produces an edge on the wrong diagonal or between the wrong +// pair, which is a silently different clustering. +// - raw and packed agree. --raw-records is the control the whole encoding +// argument rests on, so "both encodings decode to the same edges" is the +// property that makes it a control rather than a second implementation. +// - corruption is refused, not decoded. A torn tail is what an interrupted +// worker leaves, and the reader is required to stop at the last good block. + +#include "CandidateEdge.h" + +#include +#include +#include +#include +#include +#include + +const char* binary_name = "test_edgecodec"; + +static int failures = 0; + +static void check(bool condition, const std::string &what) { + if (condition == false) { + fprintf(stderr, "FAIL: %s\n", what.c_str()); + failures++; + } else { + fprintf(stdout, "ok: %s\n", what.c_str()); + } +} + +static uint64_t nextRandom(uint64_t &state) { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + return state; +} + +static bool sameEdge(const CandidateEdge &a, const CandidateEdge &b) { + return a.getRep() == b.getRep() && a.getMember() == b.getMember() && + a.diagonal == b.diagonal && a.score == b.score && a.reverseStrand == b.reverseStrand; +} + +// Encodes and decodes, returning the payload size so the two encodings can be +// compared on size as well as on content. +static bool roundTrip(const std::vector &edges, bool raw, + std::vector &out, size_t &payloadBytes) { + std::vector buffer; + EdgeBlockCodec::encode(edges, raw, 7, 3, buffer); + if (edges.empty()) { + return buffer.empty(); + } + EdgeBlockHeader header; + memcpy(&header, buffer.data(), sizeof(EdgeBlockHeader)); + if (EdgeBlockCodec::headerLooksValid(header) == false) { + return false; + } + if (header.partition != 7 || header.worker != 3 || header.recordCount != edges.size()) { + return false; + } + payloadBytes = static_cast(header.payloadBytes); + out.clear(); + return EdgeBlockCodec::decode(header, buffer.data() + sizeof(EdgeBlockHeader), out); +} + +// The encoder requires non-decreasing representatives, which the reduce +// guarantees. Everything generated here respects that. +static std::vector makeEdges(size_t count, uint64_t seed, bool extremes) { + std::vector edges; + uint64_t state = seed; + uint64_t rep = 0; + for (size_t i = 0; i < count; i++) { + CandidateEdge e; + rep += nextRandom(state) % 1000; // ascending, sometimes repeating + e.setRep(rep); + // Members on both sides of the representative: with --cov-mode 1 + // assignGroup emits reversed edges, so member < rep is a real case and + // the zigzag delta has to carry it. + const int64_t offset = static_cast(nextRandom(state) % 2000001) - 1000000; + int64_t member = static_cast(rep) + offset; + if (member < 0) { + member = 0; + } + e.setMember(static_cast(member)); + e.diagonal = static_cast(nextRandom(state) % 65536); + e.score = static_cast(nextRandom(state) % 65536); + e.reverseStrand = static_cast(nextRandom(state) & 1); + edges.push_back(e); + } + if (extremes && edges.empty() == false) { + // Every field at both ends of its range, in a block that also holds + // ordinary values, so a boundary bug cannot hide behind a special case. + edges[0].setRep(0); + edges[0].setMember(CandidateEdge::MAX_KEY); + edges[0].diagonal = INT16_MIN; + edges[0].score = 0; + edges[0].reverseStrand = 0; + CandidateEdge last; + last.setRep(CandidateEdge::MAX_KEY); + last.setMember(0); + last.diagonal = INT16_MAX; + last.score = 65535; + last.reverseStrand = 1; + edges.push_back(last); // still non-decreasing: MAX_KEY is the largest + } + return edges; +} + +static void testRoundTrip() { + const size_t counts[] = {1, 2, 17, 1000, 50000}; + bool allOk = true; + bool agreeOk = true; + bool smallerOk = true; + for (size_t c = 0; c < sizeof(counts) / sizeof(counts[0]); c++) { + std::vector edges = makeEdges(counts[c], 0x2545F4914F6CDD1DULL + c, true); + + std::vector packed, raw; + size_t packedBytes = 0, rawBytes = 0; + if (roundTrip(edges, false, packed, packedBytes) == false || + roundTrip(edges, true, raw, rawBytes) == false) { + allOk = false; + continue; + } + if (packed.size() != edges.size() || raw.size() != edges.size()) { + allOk = false; + continue; + } + for (size_t i = 0; i < edges.size(); i++) { + if (!sameEdge(packed[i], edges[i]) || !sameEdge(raw[i], edges[i])) { + allOk = false; + } + if (!sameEdge(packed[i], raw[i])) { + agreeOk = false; + } + } + // The whole point of the encoding. A change that silently stopped packing + // would pass every correctness check above. + if (packedBytes >= rawBytes) { + smallerOk = false; + } + } + check(allOk, "packed and raw both round-trip 1..50000 edges including field extremes"); + check(agreeOk, "packed and raw decode to identical edges, so --raw-records is a real control"); + check(smallerOk, "the packed payload is smaller than the fixed-width one at every size"); + + std::vector empty, out; + size_t bytes = 0; + check(roundTrip(empty, false, out, bytes), "an empty block encodes to nothing at all"); +} + +static void testMemberOnBothSides() { + // Reversed edges are not a corner case: --cov-mode 1 produces them, and the + // member delta is zigzagged specifically so they stay short. + std::vector edges; + for (int64_t d = -5; d <= 5; d++) { + CandidateEdge e; + e.setRep(1000000); + e.setMember(static_cast(1000000 + d)); + e.diagonal = static_cast(d); + e.score = 1; + e.reverseStrand = 0; + edges.push_back(e); + } + std::sort(edges.begin(), edges.end(), [](const CandidateEdge &a, const CandidateEdge &b) { + return a.getMember() < b.getMember(); + }); + std::vector out; + size_t bytes = 0; + bool ok = roundTrip(edges, false, out, bytes) && out.size() == edges.size(); + for (size_t i = 0; ok && i < edges.size(); i++) { + ok = sameEdge(out[i], edges[i]); + } + check(ok, "members below and above their representative both round-trip"); +} + +static void testCorruptionRefused() { + std::vector edges = makeEdges(500, 0x9E3779B97F4A7C15ULL, false); + std::vector good; + EdgeBlockCodec::encode(edges, false, 7, 3, good); + EdgeBlockHeader header; + memcpy(&header, good.data(), sizeof(EdgeBlockHeader)); + const uint8_t *payload = good.data() + sizeof(EdgeBlockHeader); + + std::vector out; + check(EdgeBlockCodec::decode(header, payload, out), "the untouched block decodes"); + + // A flipped payload byte must fail the checksum rather than decode into + // plausible edges. + { + std::vector broken(good); + broken[sizeof(EdgeBlockHeader) + broken.size() / 3] ^= 0x40; + out.clear(); + check(EdgeBlockCodec::decode(header, broken.data() + sizeof(EdgeBlockHeader), out) == false, + "a single flipped payload byte is refused by the checksum"); + check(out.empty(), "a refused block leaves the output untouched"); + } + + // A record count larger than the payload supports must run out of bytes + // rather than read past the end. + { + EdgeBlockHeader tooMany = header; + tooMany.recordCount = header.recordCount + 50; + out.clear(); + check(EdgeBlockCodec::decode(tooMany, payload, out) == false, + "a record count past the payload is refused"); + } + + // Trailing bytes mean the block is not what its header says. + { + EdgeBlockHeader tooFew = header; + tooFew.recordCount = header.recordCount - 10; + out.clear(); + check(EdgeBlockCodec::decode(tooFew, payload, out) == false, + "a record count short of the payload is refused"); + } + + // Header sanity, which is what stops a torn tail being read as a block. + { + EdgeBlockHeader bad = header; + bad.magic ^= 0xFFFFFFFFU; + check(EdgeBlockCodec::headerLooksValid(bad) == false, "a bad magic is rejected"); + bad = header; + bad.encoding = 99; + check(EdgeBlockCodec::headerLooksValid(bad) == false, "an unknown encoding is rejected"); + bad = header; + bad.encoding = EdgeBlockCodec::ENCODING_RAW; + check(EdgeBlockCodec::headerLooksValid(bad) == false, + "a raw header whose length disagrees with its record count is rejected"); + } +} + +// Decoding many blocks into one vector is what the align stage does per bucket. +// It is also where the quadratic reserve lived, so the growth behaviour is +// asserted rather than assumed: appending N blocks must stay linear. +static void testManyBlocksAccumulate() { + std::vector all; + std::vector out; + bool ok = true; + uint64_t rep = 0; + for (int block = 0; block < 200; block++) { + std::vector edges; + uint64_t state = 0x1234567 + block; + for (int i = 0; i < 500; i++) { + CandidateEdge e; + rep += nextRandom(state) % 100; + e.setRep(rep); + e.setMember(rep + 17); + e.diagonal = 3; + e.score = 2; + e.reverseStrand = 0; + edges.push_back(e); + all.push_back(e); + } + std::vector buffer; + EdgeBlockCodec::encode(edges, false, 1, 1, buffer); + EdgeBlockHeader header; + memcpy(&header, buffer.data(), sizeof(EdgeBlockHeader)); + if (EdgeBlockCodec::decode(header, buffer.data() + sizeof(EdgeBlockHeader), out) == false) { + ok = false; + break; + } + } + ok = ok && out.size() == all.size(); + for (size_t i = 0; ok && i < all.size(); i++) { + ok = sameEdge(out[i], all[i]); + } + check(ok, "200 blocks decoded into one vector accumulate in order and intact"); +} + +int main(int, const char**) { + testRoundTrip(); + testMemberOnBothSides(); + testCorruptionRefused(); + testManyBlocksAccumulate(); + + if (failures > 0) { + fprintf(stderr, "\n%d check(s) failed\n", failures); + return EXIT_FAILURE; + } + fprintf(stdout, "\nall checks passed\n"); + return EXIT_SUCCESS; +}