“Know how to solve every problem that has been solved.” “What I cannot create, I do not understand.” — Richard Feynman
cpp 119 lines · 4.8 KB
Raw ↗
// Step 2 -- stop reading lines; map the file and walk bytes once.
//
// Two ideas replace step 1's getline dance:
//
//   * mmap. The kernel maps the file's pages straight into our address space:
//     no read() copies into a userspace buffer, no istream layer, and the
//     "record spans multiple lines" headache disappears because there are no
//     lines -- just one contiguous byte range and one pass over it.
//   * A four-state RFC-4180 machine (FIELD_START, UNQUOTED, QUOTED,
//     QUOTE_QUOTE) that decides, byte by byte, whether we are inside quotes.
//     Record assembly and field splitting collapse into the same loop.
//
// What we deliberately KEEP from step 1: every field's bytes are still copied
// into a std::string, one += at a time. A surprise the allocation counter
// exposes: the heap traffic is already ~zero, because the field buffer is
// REUSED (clear() keeps capacity) and short fields live in the small-string
// buffer. The copies survive as pure byte traffic, not allocations -- so the
// remaining win for step 3 is to stop touching the bytes at all.
//
// Build:  c++ -std=c++20 -O2 step2_mmap.cpp -o step2
#include "csv_test_common.hpp"
#include <fcntl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>
#include <string>
#include <vector>

struct Mapped {
    const char* data = nullptr;
    size_t size = 0;
    Mapped(const char* path) {
        int fd = ::open(path, O_RDONLY);
        if (fd < 0) { std::perror(path); std::exit(1); }
        struct stat st{};
        ::fstat(fd, &st);
        size = size_t(st.st_size);
        data = (const char*)::mmap(nullptr, size, PROT_READ, MAP_PRIVATE, fd, 0);
        ::close(fd);
        if (data == MAP_FAILED) { std::perror("mmap"); std::exit(1); }
    }
    ~Mapped() { if (data) ::munmap((void*)data, size); }
};

// One pass, one state machine. on_field(str) per field, on_record() at ends.
template <typename OnField, typename OnRecord>
static void parse_mmap(const char* p, const char* end, OnField on_field, OnRecord on_record) {
    enum State { FIELD_START, UNQUOTED, QUOTED, QUOTE_QUOTE };
    State st = FIELD_START;
    std::string cur;
    bool record_open = false;

    auto end_field = [&] { on_field(cur); cur.clear(); };
    auto end_record = [&] { end_field(); on_record(); record_open = false; };

    while (p < end) {
        char c = *p++;
        record_open = true;
        switch (st) {
        case FIELD_START:
            if (c == '"') st = QUOTED;
            else if (c == ',') end_field();
            else if (c == '\n') { st = FIELD_START; end_record(); }
            else if (c == '\r') { if (p < end && *p == '\n') ++p; end_record(); st = FIELD_START; }
            else { cur += c; st = UNQUOTED; }
            break;
        case UNQUOTED:
            if (c == ',') { end_field(); st = FIELD_START; }
            else if (c == '\n') { end_record(); st = FIELD_START; }
            else if (c == '\r') { if (p < end && *p == '\n') ++p; end_record(); st = FIELD_START; }
            else cur += c;
            break;
        case QUOTED:
            if (c == '"') st = QUOTE_QUOTE;
            else cur += c;                     // includes ',', '\r', '\n'
            break;
        case QUOTE_QUOTE:                      // a quote inside a quoted field:
            if (c == '"') { cur += '"'; st = QUOTED; }        // "" -> literal "
            else if (c == ',') { end_field(); st = FIELD_START; }
            else if (c == '\n') { end_record(); st = FIELD_START; }
            else if (c == '\r') { if (p < end && *p == '\n') ++p; end_record(); st = FIELD_START; }
            break;
        }
    }
    if (record_open) end_record();             // file without trailing newline
}

static std::vector<std::vector<std::string>> parse_all_materialized(const char* path) {
    Mapped m(path);
    std::vector<std::vector<std::string>> rows;
    std::vector<std::string> rec;
    parse_mmap(m.data, m.data + m.size,
               [&](const std::string& f) { rec.push_back(f); },
               [&] { rows.push_back(rec); rec.clear(); });
    return rows;
}

int main() {
    if (!run_edge_checks("edge.csv", parse_all_materialized)) return 1;

    Mapped m("bench.csv");
    const double mb = m.size / 1e6;

    BenchResult res;
    for (int rep = 0; rep < 3; ++rep) {
        unsigned long long a0 = allocs_now();
        double t0 = now_secs();
        unsigned long long rows = 0, fields = 0, byte_sum = 0;
        parse_mmap(m.data, m.data + m.size,
                   [&](const std::string& f) { ++fields; byte_sum += f.size(); },
                   [&] { ++rows; });
        double dt = now_secs() - t0;
        if (dt < res.best_secs) res.best_secs = dt;
        res.rows = rows; res.fields = fields; res.byte_sum = byte_sum;
        res.last_pass_allocs = allocs_now() - a0;
    }
    print_bench("step2_mmap", res, mb);
    return 0;
}