diff --git a/clang/tools/dpct/gitdiff2yaml/1.ref.txt b/clang/tools/dpct/gitdiff2yaml/1.ref.txt new file mode 100644 index 000000000000..23273daa15be --- /dev/null +++ b/clang/tools/dpct/gitdiff2yaml/1.ref.txt @@ -0,0 +1,5 @@ +--- +AddFileHunks: + - NewFilePath: 'b.txt' + - NewFilePath: 'c.txt' +... diff --git a/clang/tools/dpct/gitdiff2yaml/2.ref.txt b/clang/tools/dpct/gitdiff2yaml/2.ref.txt new file mode 100644 index 000000000000..6da1bae78ec3 --- /dev/null +++ b/clang/tools/dpct/gitdiff2yaml/2.ref.txt @@ -0,0 +1,4 @@ +--- +DeleteFileHunks: + - OldFilePath: 'b.txt' +... diff --git a/clang/tools/dpct/gitdiff2yaml/3.ref.txt b/clang/tools/dpct/gitdiff2yaml/3.ref.txt new file mode 100644 index 000000000000..7579b9fee24b --- /dev/null +++ b/clang/tools/dpct/gitdiff2yaml/3.ref.txt @@ -0,0 +1,8 @@ +--- +MoveFileHunks: + - FilePath: 'b.txt' + Offset: 12 + Length: 4 + ReplacementText: "777\n" + NewFilePath: 'c.txt' +... diff --git a/clang/tools/dpct/gitdiff2yaml/4.ref.txt b/clang/tools/dpct/gitdiff2yaml/4.ref.txt new file mode 100644 index 000000000000..58fe43dd35c5 --- /dev/null +++ b/clang/tools/dpct/gitdiff2yaml/4.ref.txt @@ -0,0 +1,27 @@ +--- +ModifyFileHunks: + - FilePath: 'a.txt' + Offset: 8 + Length: 8 + ReplacementText: "111\n222\n" + - FilePath: 'a.txt' + Offset: 20 + Length: 4 + ReplacementText: "333\n" + - FilePath: 'a.txt' + Offset: 56 + Length: 0 + ReplacementText: "444\n" + - FilePath: 'a.txt' + Offset: 92 + Length: 4 + ReplacementText: "" + - FilePath: 'b.txt' + Offset: 40 + Length: 12 + ReplacementText: "" + - FilePath: 'b.txt' + Offset: 92 + Length: 0 + ReplacementText: "111\n222\n333\n" +... diff --git a/clang/tools/dpct/gitdiff2yaml/gitdiff2yaml.cpp b/clang/tools/dpct/gitdiff2yaml/gitdiff2yaml.cpp new file mode 100644 index 000000000000..4ed626d1d565 --- /dev/null +++ b/clang/tools/dpct/gitdiff2yaml/gitdiff2yaml.cpp @@ -0,0 +1,423 @@ +//===--- gitdiff2yaml.cpp -------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Usage: +// $ g++ gitdiff2yaml.cpp -o gitdiff2yaml +// $ cd /path/to/your/git/repo +// $ ./gitdiff2yaml +// This will output the clang replacements in YAML format. +// Limitation: +// (1) The workspace and the staging area should be clean before running +// this tool. +// (2) The line ending in the file should be '\n'. +//===----------------------------------------------------------------------===// + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +const std::string LineEnd = "\\n"; + +std::string execGitCommand(const std::string &CMD) { + std::array Buffer; + std::unique_ptr Pipe(popen(CMD.c_str(), "r"), pclose); + if (!Pipe) { + throw std::runtime_error("popen() failed!"); + } + + std::string Result; + while (fgets(Buffer.data(), Buffer.size(), Pipe.get()) != nullptr) { + Result += Buffer.data(); + } + return Result; +} + +struct Replacement { + std::string NewFilePath; + std::string OldFilePath; + unsigned Offset = 0; + unsigned Length = 0; + std::string ReplacementText; +}; + +struct HunkContext { + unsigned OldCurrentLine = 0; + bool InHunk = false; + bool FastForward = false; + std::string CurrentNewFilePath; + std::string CurrentOldFilePath; +}; + +bool startsWith(const std::string &Str, const std::string &Prefix) { + return Str.size() >= Prefix.size() && + Str.compare(0, Prefix.size(), Prefix) == 0; +} + +bool parseHunkHeader(const std::string &Line, HunkContext &HC) { + const std::string HunkHeaderPrefix = "@@ -"; + if (!startsWith(Line, HunkHeaderPrefix)) + return false; + + // E.g., + // @@ -0,0 +1,3 @@ + // ^ + // |-- OldEnd + size_t OldEnd = Line.find(' ', HunkHeaderPrefix.size()); + std::string OldPart = + Line.substr(HunkHeaderPrefix.size(), OldEnd - HunkHeaderPrefix.size()); + size_t Comma = OldPart.find(','); + if (Comma == std::string::npos) { + throw std::runtime_error("Invalid hunk header format: " + Line); + } + HC.OldCurrentLine = std::stoi(OldPart.substr(0, Comma)); + return true; +} + +// vec[0] is -1, which is a placeholder. +// vec[i] /*i is the line number (1-based)*/ is the offset at the line +// beginning. +// Assumption: the line ending in the file is '\n'. +std::vector calculateOldOffset(const std::string &OldFileContent) { + std::vector Ret; + Ret.push_back(-1); // Placeholder for Ret[0]. + std::istringstream OldFileStream(OldFileContent); + std::string Line; + unsigned Offset = 0; + + while (std::getline(OldFileStream, Line)) { + Ret.push_back(Offset); + Offset += Line.size() + + 1; // std::getline does not include the newline character. And we + // assume the line ending is '\n'. So add 1 here. + } + + return Ret; +} + +// 1. Assume the line ending in the file is '\n'. +// 2. The pair (---, +++) may occurs multiple times in one hunk, so we use a +// variable to save the delete (-) operation. The continuous delete operations +// are treated as one operation. +// 3. After the delete operation, if the next line is one or more '+' +// operations, we make them as a replace-replacement. If the next line is a +// context line, the delete operation is a delete-replacement. Then clear the +// variable. +// 4. If we meet insertions ('+') when the variable is empty, we treat it as an +// insert-replacement. +void processHunkBody(const std::string &Line, HunkContext &Ctx, + std::vector &Repls, + const std::vector &CurrentOldFileOffset) { + static std::optional< + std::pair> + DeleteInfo; + static std::optional< + std::pair> + AddInfo; + + auto addRepl = [&]() { + Replacement R; + if (DeleteInfo.has_value() && AddInfo.has_value()) { + // replace-replacement + R.OldFilePath = Ctx.CurrentOldFilePath; + R.NewFilePath = Ctx.CurrentNewFilePath; + R.Length = DeleteInfo->second; + R.Offset = CurrentOldFileOffset[DeleteInfo->first]; + R.ReplacementText = AddInfo->second; + DeleteInfo.reset(); + AddInfo.reset(); + } else if (DeleteInfo.has_value()) { + // delete-replacement + R.OldFilePath = Ctx.CurrentOldFilePath; + R.NewFilePath = Ctx.CurrentNewFilePath; + R.Length = DeleteInfo->second; + R.Offset = CurrentOldFileOffset[DeleteInfo->first]; + R.ReplacementText = ""; + DeleteInfo.reset(); + } else if (AddInfo.has_value()) { + // insert-replacement + R.OldFilePath = Ctx.CurrentOldFilePath; + R.NewFilePath = Ctx.CurrentNewFilePath; + R.Length = 0; + R.Offset = CurrentOldFileOffset[AddInfo->first]; + R.ReplacementText = AddInfo->second; + AddInfo.reset(); + } + Repls.push_back(R); + }; + + // Hunk end + if (Line.empty()) { + addRepl(); + Ctx.InHunk = false; + return; + } + + switch (Line[0]) { + case ' ': { + addRepl(); + Ctx.OldCurrentLine++; + break; + } + case '-': { + if (!DeleteInfo.has_value()) { + auto Item = std::pair( + Ctx.OldCurrentLine, + Line.length()); // +1 for the newline character, -1 for the + // '-' in the line beginng + DeleteInfo = Item; + } else { + DeleteInfo->second += + (Line.length()); // +1 for the newline character, -1 for the + // '-' in the line beginng + } + Ctx.OldCurrentLine++; + break; + } + case '+': { + if (!AddInfo.has_value()) { + auto Item = std::pair(Ctx.OldCurrentLine, + Line.substr(1) + LineEnd); + AddInfo = Item; + } else { + AddInfo->second += (Line.substr(1) + LineEnd); + } + break; + } + } +} + +std::vector parseDiff(const std::string &diffOutput, + const std::string &RepoRoot) { + std::vector replacements; + std::istringstream iss(diffOutput); + std::string line; + + HunkContext HC; + std::vector CurrentOldFileOffset; + + while (std::getline(iss, line)) { + if (startsWith(line, "diff --git")) { + HC.FastForward = false; + continue; + } + if (HC.FastForward) + continue; + + if (startsWith(line, "---")) { + HC.CurrentOldFilePath = + line.substr(4) == "/dev/null" ? "/dev/null" : line.substr(6); + if (HC.CurrentOldFilePath != "/dev/null") { + std::ifstream FileStream(RepoRoot + "/" + HC.CurrentOldFilePath); + if (!FileStream.is_open()) { + throw std::runtime_error("Failed to open file: " + RepoRoot + "/" + + HC.CurrentOldFilePath); + } + std::stringstream Buffer; + Buffer << FileStream.rdbuf(); + CurrentOldFileOffset = calculateOldOffset(Buffer.str()); + } + continue; + } + if (startsWith(line, "+++")) { + HC.CurrentNewFilePath = + line.substr(4) == "/dev/null" ? "/dev/null" : line.substr(6); + if (HC.CurrentOldFilePath == "/dev/null" || + HC.CurrentNewFilePath == "/dev/null") { + HC.FastForward = true; + Replacement R; + R.OldFilePath = HC.CurrentOldFilePath; + R.NewFilePath = HC.CurrentNewFilePath; + replacements.emplace_back(R); + } + continue; + } + + if (parseHunkHeader(line, HC)) { + // Hunk start + HC.InHunk = true; + continue; + } + + if (HC.InHunk) { + processHunkBody(line, HC, replacements, CurrentOldFileOffset); + continue; + } + } + + return replacements; +} + +struct ModifyHunk { + std::string FilePath; + unsigned Offset = 0; + unsigned Length = 0; + std::string ReplacementText; +}; + +struct AddHunk { + std::string NewFilePath; +}; + +struct DeleteHunk { + std::string OldFilePath; +}; + +struct MoveHunk { + std::string FilePath; + unsigned Offset = 0; + unsigned Length = 0; + std::string ReplacementText; + std::string NewFilePath; +}; + +void printYaml(std::ostream &stream, const std::vector &Repls) { + std::vector ModifyHunks; + std::vector AddHunks; + std::vector DeleteHunks; + std::vector MoveHunks; + + for (const auto &R : Repls) { + if (R.OldFilePath == "/dev/null" && R.NewFilePath != "/dev/null") { + // Add replacement + AddHunk AH; + AH.NewFilePath = R.NewFilePath; + AddHunks.push_back(AH); + continue; + } + if (R.OldFilePath != "/dev/null" && R.NewFilePath == "/dev/null") { + // Delete replacement + DeleteHunk DH; + DH.OldFilePath = R.OldFilePath; + DeleteHunks.push_back(DH); + continue; + } + if (R.OldFilePath == R.NewFilePath && R.OldFilePath != "/dev/null") { + // Modify replacement + ModifyHunk MH; + MH.FilePath = R.OldFilePath; + MH.Offset = R.Offset; + MH.Length = R.Length; + MH.ReplacementText = R.ReplacementText; + ModifyHunks.push_back(MH); + continue; + } + if (R.OldFilePath != R.NewFilePath) { + // Move replacement + MoveHunk MH; + MH.FilePath = R.OldFilePath; + MH.Offset = R.Offset; + MH.Length = R.Length; + MH.ReplacementText = R.ReplacementText; + MH.NewFilePath = R.NewFilePath; + MoveHunks.push_back(MH); + continue; + } + throw std::runtime_error("Invalid replacement: " + R.OldFilePath + " -> " + + R.NewFilePath); + } + + stream << "---" << std::endl; + if (!ModifyHunks.empty()) + stream << "ModifyFileHunks:" << std::endl; + for (const auto &H : ModifyHunks) { + stream << " - FilePath: " << "'" << H.FilePath << "'" << std::endl; + stream << " Offset: " << H.Offset << std::endl; + stream << " Length: " << H.Length << std::endl; + stream << " ReplacementText: " << "\"" << H.ReplacementText << "\"" + << std::endl; + } + if (!AddHunks.empty()) + stream << "AddFileHunks:" << std::endl; + for (const auto &H : AddHunks) { + stream << " - NewFilePath: " << "'" << H.NewFilePath << "'" + << std::endl; + } + if (!DeleteHunks.empty()) + stream << "DeleteFileHunks:" << std::endl; + for (const auto &H : DeleteHunks) { + stream << " - OldFilePath: " << "'" << H.OldFilePath << "'" + << std::endl; + } + if (!MoveHunks.empty()) + stream << "MoveFileHunks:" << std::endl; + for (const auto &H : MoveHunks) { + stream << " - FilePath: " << "'" << H.FilePath << "'" << std::endl; + stream << " Offset: " << H.Offset << std::endl; + stream << " Length: " << H.Length << std::endl; + stream << " ReplacementText: " << "\"" << H.ReplacementText << "\"" + << std::endl; + stream << " NewFilePath: " << "'" << H.NewFilePath << "'" + << std::endl; + } + stream << "..." << std::endl; +} + +} // namespace + +int main(int argc, char *argv[]) { + if (argc != 2 && argc != 4) { + std::cerr << "Usage: gitdiff2yaml [-o outputfile]" + << std::endl; + return 1; + } + bool OutputToFile = false; + if (argc == 4) { + if (std::string(argv[2]) != "-o") { + std::cerr << "Invalid option: " << argv[2] << std::endl; + return 1; + } + OutputToFile = true; + } + + std::string OldCommitID = argv[1]; + + std::string RepoRoot = execGitCommand("git rev-parse --show-toplevel"); + RepoRoot = RepoRoot.substr(0, RepoRoot.size() - 1); // Remove the last '\n' + + std::string NewCommitID = execGitCommand("git log -1 --format=\"%H\""); + std::string DiffOutput = execGitCommand("git diff " + OldCommitID); + + execGitCommand("git reset --hard " + OldCommitID); + std::vector Repls = parseDiff(DiffOutput, RepoRoot); + + // Erase emtpy replacements + Repls.erase(std::remove_if(Repls.begin(), Repls.end(), + [](Replacement x) { + return (x.NewFilePath == "" && + x.OldFilePath == "" && x.Offset == 0 && + x.Length == 0 && + x.ReplacementText == ""); + }), + Repls.end()); + + if (OutputToFile) { + std::ofstream OutFile(argv[3]); + if (!OutFile.is_open()) { + std::cerr << "Failed to open output file: " << argv[3] << std::endl; + return 1; + } + printYaml(OutFile, Repls); + OutFile.close(); + } else { + printYaml(std::cout, Repls); + } + + execGitCommand("git reset --hard " + NewCommitID); + + return 0; +} diff --git a/clang/tools/dpct/gitdiff2yaml/test.py b/clang/tools/dpct/gitdiff2yaml/test.py new file mode 100644 index 000000000000..7c4dc291cb5b --- /dev/null +++ b/clang/tools/dpct/gitdiff2yaml/test.py @@ -0,0 +1,101 @@ +# ===----- test.py --------------------------------------------------------=== # +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ===----------------------------------------------------------------------=== # + +from pathlib import Path +import os +import shutil +import subprocess + +def run_command(cmd): + try: + result = subprocess.run( + cmd, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + return result.stdout + except subprocess.CalledProcessError as e: + print(f"Command fail: {e.stderr}") + return None + +def compare_with_diff(ref_file): + result = subprocess.run( + ["diff", "-u", ref_file, "temp.txt"], + capture_output=True, + text=True + ) + + if result.returncode == 0: + return True + else: + print("Found difference:\n" + result.stdout) + return False + +# Clean up previous test directory if it exists +test_dir = Path("tests") +if test_dir.exists() and test_dir.is_dir(): + shutil.rmtree(test_dir) + +# Check if gitdiff2yaml exists +gitdiff2yaml_path = Path("gitdiff2yaml") +if not gitdiff2yaml_path.exists(): + print("gitdiff2yaml does not exist. Please build it first.") + exit(1) + +run_command(["unzip", "tests.zip"]) +os.chdir("tests") + +# Test 1 +os.chdir("1") +old_commit_id = run_command(["git", "rev-parse", "HEAD~1"]).strip() +result = run_command(["../../gitdiff2yaml", old_commit_id, "-o", "temp.txt"]) +test1_res = compare_with_diff("../../1.ref.txt") +if test1_res: + print("Test 1 passed") +else: + print("Test 1 failed") +os.chdir("..") + +# Test 2 +os.chdir("2") +old_commit_id = run_command(["git", "rev-parse", "HEAD~1"]).strip() +result = run_command(["../../gitdiff2yaml", old_commit_id, "-o", "temp.txt"]) +test1_res = compare_with_diff("../../2.ref.txt") +if test1_res: + print("Test 2 passed") +else: + print("Test 2 failed") +os.chdir("..") + +# Test 3 +os.chdir("3") +old_commit_id = run_command(["git", "rev-parse", "HEAD~1"]).strip() +result = run_command(["../../gitdiff2yaml", old_commit_id, "-o", "temp.txt"]) +test1_res = compare_with_diff("../../3.ref.txt") +if test1_res: + print("Test 3 passed") +else: + print("Test 3 failed") +os.chdir("..") + +# Test 4 +os.chdir("4") +old_commit_id = run_command(["git", "rev-parse", "HEAD~1"]).strip() +result = run_command(["../../gitdiff2yaml", old_commit_id, "-o", "temp.txt"]) +test1_res = compare_with_diff("../../4.ref.txt") +if test1_res: + print("Test 4 passed") +else: + print("Test 4 failed") +os.chdir("..") + +# Clean up +os.chdir("..") +shutil.rmtree(test_dir) diff --git a/clang/tools/dpct/gitdiff2yaml/tests.zip b/clang/tools/dpct/gitdiff2yaml/tests.zip new file mode 100644 index 000000000000..dec688f804e8 Binary files /dev/null and b/clang/tools/dpct/gitdiff2yaml/tests.zip differ