data_gen.py

# this script uses the dataset generated by the EALink paper and
# transforms it into pairs of (issue_url, github_url, commit_hash)

import logging
import os

import gdown
import pandas as pd
import py7zr
from dateutil.parser import parse

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)


def download_ealink_dataset():
    """
    Downloads the dataset from Google Drive.

    Returns:
        str: Path to the downloaded dataset file
    """
    # Create data directory if it doesn't exist
    data_dir = os.path.join(
        os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data"
    )
    if not os.path.exists(data_dir):
        logger.info(f"Creating data directory at {data_dir}")
        os.makedirs(data_dir)

    # URL for the finaldata.7z file in Google Drive from the EALink paper
    url = "https://drive.google.com/drive/folders/1coZbAtOYGPVQQdjf2MnykMFpfLyhw1JZ"

    # Path to save the downloaded file
    output_path = os.path.join(data_dir, "finaldata.7z")

    # Download the file if it doesn't exist
    if not os.path.exists(output_path):
        logger.info(f"Downloading dataset from {url} to {output_path}")
        try:
            gdown.download_folder(url, output=data_dir, quiet=False)

            # Check if the file was downloaded correctly
            if not os.path.exists(output_path):
                # Try direct file download if folder download didn't work
                file_id = "1coZbAtOYGPVQQdjf2MnykMFpfLyhw1JZ"
                gdown.download(
                    f"https://drive.google.com/uc?id={file_id}",
                    output_path,
                    quiet=False,
                )
        except Exception as e:
            logger.error(f"Error downloading the dataset: {e}")
            raise
    else:
        logger.info(f"Dataset already downloaded at {output_path}")

    return output_path


def extract_dataset(file_path):
    """
    Extracts the 7z dataset file.

    Args:
        file_path (str): Path to the downloaded 7z file

    Returns:
        str: Path to the extracted dataset
    """
    data_dir = os.path.dirname(file_path)
    extract_path = os.path.join(data_dir, "ealink_raw_extracted")

    if not os.path.exists(extract_path):
        logger.info(f"Extracting dataset to {extract_path}")
        try:
            os.makedirs(extract_path, exist_ok=True)
            with py7zr.SevenZipFile(file_path, mode="r") as z:
                z.extractall(path=extract_path)
            logger.info("Dataset extracted successfully")
        except Exception as e:
            logger.error(f"Error extracting the dataset: {e}")
            raise
    else:
        logger.info(f"Dataset already extracted at {extract_path}")

    return extract_path


def download_and_extract_dataset():
    """
    Downloads the dataset from Google Drive and extracts it to the data directory.
    """
    file_path = download_ealink_dataset()
    extract_path = extract_dataset(file_path)
    return extract_path


def is_issue_old(row):
    created_at = parse(row["create_date"])
    closed_at = parse(row["update_date"])
    return (closed_at - created_at).days > 365


def is_issue_new(row):
    return not is_issue_old(row)


def create_dataset(file_path):
    data = pd.read_csv(file_path, encoding="utf-8")

    # Filter all true links
    data = data[data["label"] == 1]

    # Drop unnecessary columns
    data = data[["issue_id", "repo", "commitid", "label", "create_date", "update_date"]]

    dataset = pd.DataFrame(
        {
            "issue_url": data["issue_id"].apply(
                lambda key: f"https://issues.apache.org/jira/rest/api/2/issue/{key}"
            ),
            "commit_hash": data["commitid"],
            "repo_url": data["repo"].apply(
                lambda key: f"https://github.com/apache/{key}"
            ),
            "create_date": data["create_date"],
            "update_date": data["update_date"],
        }
    )
    # return dataset
    return dataset.loc[dataset.apply(is_issue_new, axis=1)]


def prepare_ealink_dataset():
    """
    Prepares the EALink dataset by downloading and extracting it.
    """

    dataset_path = download_and_extract_dataset()

    output_dir = os.path.join(
        os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
        "data",
        "ealink",
        "csv",
    )
    os.makedirs(output_dir, exist_ok=True)

    # Path to the CSV files
    csv_files_path = os.path.join(dataset_path, "rawdata")

    # for eash csv file, read it and create a dataframe, the print the columns
    for file in os.listdir(csv_files_path):
        if file.endswith(".csv"):
            file_path = os.path.join(csv_files_path, file)

            ds = create_dataset(file_path)

            output_file = os.path.join(output_dir, file)
            # persist to csv with the headers
            ds.index.name = "index"
            ds.to_csv(output_file, header=True)
            logger.info(f"Processed dataset saved to {output_file}")

    return output_dir


# Execute the download and extraction
if __name__ == "__main__":
    dataset_path = prepare_ealink_dataset()
    logger.info(f"Dataset ready at {dataset_path}")