{

std::string hash;

std::string author;

LocalDateTime time{};

std::string message;

uint32_t files_added{};

uint32_t files_deleted{};

uint32_t files_renamed{};

uint32_t files_modified{};

uint32_t lines_added{};

uint32_t lines_deleted{};

uint32_t hunks_added{};

uint32_t hunks_removed{};

uint32_t hunks_changed{};

void writeTextWithoutNewline(WriteBuffer & out) const

{

writeEscapedString(hash, out);

writeChar('\t', out);

writeEscapedString(author, out);

writeChar('\t', out);

writeText(time, out);

writeChar('\t', out);

writeEscapedString(message, out);

writeChar('\t', out);

writeText(files_added, out);

writeChar('\t', out);

writeText(files_deleted, out);

writeChar('\t', out);

writeText(files_renamed, out);

writeChar('\t', out);

writeText(files_modified, out);

writeChar('\t', out);

writeText(lines_added, out);

writeChar('\t', out);

writeText(lines_deleted, out);

writeChar('\t', out);

writeText(hunks_added, out);

writeChar('\t', out);

writeText(hunks_removed, out);

writeChar('\t', out);

writeText(hunks_changed, out);

}

{

Add,

Delete,

Modify,

Rename,

Copy,

Type,

{

switch (type)

{

case FileChangeType::Add: writeString("Add", out); break;

case FileChangeType::Delete: writeString("Delete", out); break;

case FileChangeType::Modify: writeString("Modify", out); break;

case FileChangeType::Rename: writeString("Rename", out); break;

case FileChangeType::Copy: writeString("Copy", out); break;

case FileChangeType::Type: writeString("Type", out); break;

}

{

FileChangeType change_type{};

std::string path;

std::string old_path;

std::string file_extension;

uint32_t lines_added{};

uint32_t lines_deleted{};

uint32_t hunks_added{};

uint32_t hunks_removed{};

uint32_t hunks_changed{};

void writeTextWithoutNewline(WriteBuffer & out) const

{

writeText(change_type, out);

writeChar('\t', out);

writeEscapedString(path, out);

writeChar('\t', out);

writeEscapedString(old_path, out);

writeChar('\t', out);

writeEscapedString(file_extension, out);

writeChar('\t', out);

writeText(lines_added, out);

writeChar('\t', out);

writeText(lines_deleted, out);

writeChar('\t', out);

writeText(hunks_added, out);

writeChar('\t', out);

writeText(hunks_removed, out);

writeChar('\t', out);

writeText(hunks_changed, out);

}

{

Empty,

Comment,

Punct,

Code,

{

switch (type)

{

case LineType::Empty: writeString("Empty", out); break;

case LineType::Comment: writeString("Comment", out); break;

case LineType::Punct: writeString("Punct", out); break;

case LineType::Code: writeString("Code", out); break;

}

{

int8_t sign{}; /// 1 if added, -1 if deleted

uint32_t line_number_old{};

uint32_t line_number_new{};

uint32_t hunk_num{}; /// ordinal number of hunk in diff, starting with 0

uint32_t hunk_start_line_number_old{};

uint32_t hunk_start_line_number_new{};

uint32_t hunk_lines_added{};

uint32_t hunk_lines_deleted{};

std::string hunk_context; /// The context (like a line with function name) as it is calculated by git

std::string line; /// Line content without leading whitespaces

uint8_t indent{}; /// The number of leading whitespaces or tabs * 4

LineType line_type{};

/// Information from the history (blame).

std::string prev_commit_hash;

std::string prev_author;

LocalDateTime prev_time{};

/** Classify line to empty / code / comment / single punctuation char.

void setLineInfo(std::string full_line)

{

UInt8 num_spaces = 0;

const char * pos = full_line.data();

const char * end = pos + full_line.size();

while (pos < end)

{

if (*pos == ' ')

++num_spaces;

else if (*pos == '\t')

num_spaces += 4;

else

break;

++pos;

}

indent = std::min<UInt8>(255U, num_spaces);

line.assign(pos, end);

if (pos == end)

{

line_type = LineType::Empty;

}

else if (pos + 1 < end

&& ((pos[0] == '/' && (pos[1] == '/' || pos[1] == '*'))

|| (pos[0] == '*' && pos[1] == ' ') /// This is not precise.

|| (pos[0] == '#' && pos[1] == ' ')))

{

line_type = LineType::Comment;

}

else

{

while (pos < end)

{

if (isAlphaNumericASCII(*pos))

{

line_type = LineType::Code;

break;

}

++pos;

}

if (pos == end)

line_type = LineType::Punct;

}

}

void writeTextWithoutNewline(WriteBuffer & out) const

{

writeText(sign, out);

writeChar('\t', out);

writeText(line_number_old, out);

writeChar('\t', out);

writeText(line_number_new, out);

writeChar('\t', out);

writeText(hunk_num, out);

writeChar('\t', out);

writeText(hunk_start_line_number_old, out);

writeChar('\t', out);

writeText(hunk_start_line_number_new, out);

writeChar('\t', out);

writeText(hunk_lines_added, out);

writeChar('\t', out);

writeText(hunk_lines_deleted, out);

writeChar('\t', out);

writeEscapedString(hunk_context, out);

writeChar('\t', out);

writeEscapedString(line, out);

writeChar('\t', out);

writeText(indent, out);

writeChar('\t', out);

writeText(line_type, out);

writeChar('\t', out);

writeEscapedString(prev_commit_hash, out);

writeChar('\t', out);

writeEscapedString(prev_author, out);

writeChar('\t', out);

writeText(prev_time, out);

}

{

explicit FileDiff(FileChange file_change_) : file_change(file_change_) {}

FileChange file_change;

LineChanges line_changes;

{

while (!buf.eof())

{

char * next_pos = find_first_symbols<'\t', '\n', ' '>(buf.position(), buf.buffer().end());

buf.position() = next_pos;

if (!buf.hasPendingData())

continue;

if (*buf.position() == '\t' || *buf.position() == '\n' || *buf.position() == ' ')

return;

}

{

while (!buf.eof())

{

char * next_pos = find_first_symbols<'\n'>(buf.position(), buf.buffer().end());

buf.position() = next_pos;

if (!buf.hasPendingData())

continue;

if (*buf.position() == '\n')

{

++buf.position();

return;

}

}

{

s.clear();

while (!buf.eof())

{

char * next_pos = find_first_symbols<'\n'>(buf.position(), buf.buffer().end());

s.append(buf.position(), next_pos - buf.position());

buf.position() = next_pos;

if (!buf.hasPendingData())

continue;

if (*buf.position() == '\n')

{

++buf.position();

return;

}

}

{

WriteBufferFromFile commits{"commits.tsv"};

WriteBufferFromFile file_changes{"file_changes.tsv"};

WriteBufferFromFile line_changes{"line_changes.tsv"};

void appendCommit(const Commit & commit, const CommitDiff & files)

{

/// commits table

{

auto & out = commits;

commit.writeTextWithoutNewline(out);

writeChar('\n', out);

}

for (const auto & elem : files)

{

const FileChange & file_change = elem.second.file_change;

/// file_changes table

{

auto & out = file_changes;

file_change.writeTextWithoutNewline(out);

writeChar('\t', out);

commit.writeTextWithoutNewline(out);

writeChar('\n', out);

}

/// line_changes table

for (const auto & line_change : elem.second.line_changes)

{

auto & out = line_changes;

line_change.writeTextWithoutNewline(out);

writeChar('\t', out);

file_change.writeTextWithoutNewline(out);

writeChar('\t', out);

commit.writeTextWithoutNewline(out);

writeChar('\n', out);

}

}

}

void finalize()

{

commits.finalize();

file_changes.finalize();

line_changes.finalize();

}

{

bool skip_commits_without_parents = true;

bool skip_commits_with_duplicate_diffs = true;

size_t threads = 1;

std::optional<re2::RE2> skip_paths;

std::optional<re2::RE2> skip_commits_with_messages;

std::unordered_set<std::string> skip_commits;

std::optional<size_t> diff_size_limit;

std::string stop_after_commit;

explicit Options(const po::variables_map & options)

{

skip_commits_without_parents = options["skip-commits-without-parents"].as<bool>();

skip_commits_with_duplicate_diffs = options["skip-commits-with-duplicate-diffs"].as<bool>();

threads = options["threads"].as<size_t>();

if (options.contains("skip-paths"))

{

skip_paths.emplace(options["skip-paths"].as<std::string>());

}

if (options.contains("skip-commits-with-messages"))

{

skip_commits_with_messages.emplace(options["skip-commits-with-messages"].as<std::string>());

}

if (options.contains("skip-commit"))

{

auto vec = options["skip-commit"].as<std::vector<std::string>>();

skip_commits.insert(vec.begin(), vec.end());

}

if (options.contains("diff-size-limit"))

{

diff_size_limit = options["diff-size-limit"].as<size_t>();

}

if (options.contains("stop-after-commit"))

{

stop_after_commit = options["stop-after-commit"].as<std::string>();

}

}

{

using Lines = std::list<Commit>;

Lines lines;

/// We walk through this list adding or removing lines.

Lines::iterator it;

size_t current_idx = 1;

FileBlame()

{

it = lines.begin();

}

/// This is important when file was copied or renamed.

FileBlame & operator=(const FileBlame & rhs)

{

if (&rhs == this)

return *this;

lines = rhs.lines;

it = lines.begin();

current_idx = 1;

return *this;

}

FileBlame(const FileBlame & rhs)

{

*this = rhs;

}

/// Move iterator to requested line or stop at the end.

void walk(uint32_t num)

{

while (current_idx < num && it != lines.end())

{

++current_idx;

++it;

}

while (current_idx > num)

{

--current_idx;

--it;

}

}

const Commit * find(uint32_t num)

{

walk(num);

if (current_idx == num && it != lines.end())

return &*it;

return {};

}

void addLine(uint32_t num, Commit commit)

{

walk(num);

/// If the inserted line is over the end of file, we insert empty lines before it.

while (it == lines.end() && current_idx < num)

{

lines.emplace_back();

++current_idx;

}

it = lines.insert(it, commit);

}

void removeLine(uint32_t num)

{

walk(num);

if (current_idx == num && it != lines.end())

it = lines.erase(it);

}

{

/// Renames and copies.

for (auto & elem : file_changes)

{

auto & file = elem.second.file_change;

if (!file.old_path.empty() && file.path != file.old_path)

snapshot[file.path] = snapshot[file.old_path];

}

for (auto & elem : file_changes)

{

FileBlame & file_snapshot = snapshot[elem.first];

std::unordered_map<uint32_t, Commit> deleted_lines;

/// Obtain blame info from previous state of the snapshot

for (auto & line_change : elem.second.line_changes)

{

if (line_change.sign == -1)

{

if (const Commit * prev_commit = file_snapshot.find(line_change.line_number_old);

prev_commit && prev_commit->time <= commit.time)

{

line_change.prev_commit_hash = prev_commit->hash;

line_change.prev_author = prev_commit->author;

line_change.prev_time = prev_commit->time;

deleted_lines[line_change.line_number_old] = *prev_commit;

}

else

{

// std::cerr << "Did not find line " << line_change.line_number_old << " from file " << elem.first << ": " << line_change.line << "\n";

}

}

else if (line_change.sign == 1)

{

uint32_t this_line_in_prev_commit = line_change.hunk_start_line_number_old

+ (line_change.line_number_new - line_change.hunk_start_line_number_new);

if (deleted_lines.contains(this_line_in_prev_commit))

{

const auto & prev_commit = deleted_lines[this_line_in_prev_commit];

if (prev_commit.time <= commit.time)

{

line_change.prev_commit_hash = prev_commit.hash;

line_change.prev_author = prev_commit.author;

line_change.prev_time = prev_commit.time;

}

}

}

}

/// Update the snapshot

for (const auto & line_change : elem.second.line_changes)

{

if (line_change.sign == -1)

{

file_snapshot.removeLine(line_change.line_number_new);

}

else if (line_change.sign == 1)

{

file_snapshot.addLine(line_change.line_number_new, commit);

}

}

}

{

SipHash hasher;

for (const auto & elem : file_changes)

{

hasher.update(elem.second.file_change.change_type);

hasher.update(elem.second.file_change.old_path.size());

hasher.update(elem.second.file_change.old_path);

hasher.update(elem.second.file_change.path.size());

hasher.update(elem.second.file_change.path);

hasher.update(elem.second.line_changes.size());

for (const auto & line_change : elem.second.line_changes)

{

hasher.update(line_change.sign);

hasher.update(line_change.line_number_old);

hasher.update(line_change.line_number_new);

hasher.update(line_change.indent);

hasher.update(line_change.line.size());

hasher.update(line_change.line);

}

}

UInt128 hash_of_diff;

hasher.get128(hash_of_diff.items[0], hash_of_diff.items[1]);

return hash_of_diff;

ReadBuffer & in,

const Options & options,

Commit & commit,

CommitDiff & file_changes)

{

while (checkChar(':', in))

{

FileChange file_change;

/// We don't care about file mode and content hashes.

for (size_t i = 0; i < 4; ++i)

{

skipUntilWhitespace(in);

skipWhitespaceIfAny(in);

}

char change_type = {};

readChar(change_type, in);

/// For rename and copy there is a number called "score". We ignore it.

int score = {};

switch (change_type)

{

case 'A':

file_change.change_type = FileChangeType::Add;

++commit.files_added;

break;

case 'D':

file_change.change_type = FileChangeType::Delete;

++commit.files_deleted;

break;

case 'M':

file_change.change_type = FileChangeType::Modify;

++commit.files_modified;

break;

case 'R':

file_change.change_type = FileChangeType::Rename;

++commit.files_renamed;

readText(score, in);

break;

case 'C':

file_change.change_type = FileChangeType::Copy;

readText(score, in);

break;

case 'T':

file_change.change_type = FileChangeType::Type;

break;

default:

throw Exception(ErrorCodes::INCORRECT_DATA, "Unexpected file change type: {}", change_type);

}

skipWhitespaceIfAny(in);

if (change_type == 'R' || change_type == 'C')

{

readText(file_change.old_path, in);

skipWhitespaceIfAny(in);

readText(file_change.path, in);

}

else

{

readText(file_change.path, in);

}

file_change.file_extension = std::filesystem::path(file_change.path).extension();

/// It gives us extension in form of '.cpp'. There is a reason for it but we remove initial dot for simplicity.

if (!file_change.file_extension.empty() && file_change.file_extension.front() == '.')

file_change.file_extension = file_change.file_extension.substr(1, std::string::npos);

assertChar('\n', in);

if (!(options.skip_paths && re2::RE2::PartialMatch(file_change.path, *options.skip_paths)))

{

file_changes.emplace(

file_change.path,

FileDiff(file_change));

}

}

ReadBuffer & in,

std::optional<size_t> size_limit,

Commit & commit,

CommitDiff & file_changes)

{

std::string old_file_path;

std::string new_file_path;

FileDiff * file_change_and_line_changes = nullptr;

LineChange line_change;

/// Diffs for every file in form of

/// --- a/src/Storages/StorageReplicatedMergeTree.cpp

/// +++ b/src/Storages/StorageReplicatedMergeTree.cpp

/// @@ -1387,2 +1387 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry)

/// - table_lock, entry.create_time, reserved_space, entry.deduplicate,

/// - entry.force_ttl);

/// + table_lock, entry.create_time, reserved_space, entry.deduplicate);

size_t diff_size = 0;

while (!in.eof())

{

if (checkString("@@ ", in))

{

if (!file_change_and_line_changes)

{

auto file_name = new_file_path.empty() ? old_file_path : new_file_path;

auto it = file_changes.find(file_name);

if (file_changes.end() != it)

file_change_and_line_changes = &it->second;

}

if (file_change_and_line_changes)

{

uint32_t old_lines = 1;

uint32_t new_lines = 1;

assertChar('-', in);

readText(line_change.hunk_start_line_number_old, in);

if (checkChar(',', in))

readText(old_lines, in);

assertString(" +", in);

readText(line_change.hunk_start_line_number_new, in);

if (checkChar(',', in))

readText(new_lines, in);

/// This is needed to simplify the logic of updating snapshot:

/// When all lines are removed we can treat it as repeated removal of line with number 1.

if (line_change.hunk_start_line_number_new == 0)

line_change.hunk_start_line_number_new = 1;

assertString(" @@", in);

if (checkChar(' ', in))

readStringUntilNextLine(line_change.hunk_context, in);

else

assertChar('\n', in);

line_change.hunk_lines_added = new_lines;

line_change.hunk_lines_deleted = old_lines;

++line_change.hunk_num;

line_change.line_number_old = line_change.hunk_start_line_number_old;

line_change.line_number_new = line_change.hunk_start_line_number_new;

if (old_lines && new_lines)

{

++commit.hunks_changed;

++file_change_and_line_changes->file_change.hunks_changed;

}

else if (old_lines)

{

++commit.hunks_removed;

++file_change_and_line_changes->file_change.hunks_removed;

}

else if (new_lines)

{

++commit.hunks_added;

++file_change_and_line_changes->file_change.hunks_added;

}

}

}

else if (checkChar('-', in))

{

if (checkString("-- ", in))

{

if (checkString("a/", in))

{

readStringUntilNextLine(old_file_path, in);

line_change = LineChange{};

file_change_and_line_changes = nullptr;

}

else if (checkString("/dev/null", in))

{

old_file_path.clear();

assertChar('\n', in);

line_change = LineChange{};

file_change_and_line_changes = nullptr;

}

else

skipUntilNextLine(in); /// Actually it can be the line in diff. Skip it for simplicity.

}

else

{

++diff_size;

if (file_change_and_line_changes)

{

++commit.lines_deleted;

++file_change_and_line_changes->file_change.lines_deleted;

line_change.sign = -1;

readStringUntilNextLine(line_change.line, in);