diff --git a/.codacy/codacy.yaml b/.codacy/codacy.yaml index efc9a772..74b95e68 100644 --- a/.codacy/codacy.yaml +++ b/.codacy/codacy.yaml @@ -1,4 +1,6 @@ runtimes: - node@22.2.0 + - python@3.9.21 tools: - eslint@9.3.0 + - pylint@2.13.9 diff --git a/cmd/analyze.go b/cmd/analyze.go index dd5399fa..f8aecf90 100644 --- a/cmd/analyze.go +++ b/cmd/analyze.go @@ -198,8 +198,24 @@ var analyzeCmd = &cobra.Command{ switch toolToAnalyze { case "eslint": // nothing + case "pylint": + pylint := config.Config.Tools()["pylint"] + if pylint == nil { + log.Fatal("Pylint is not installed. Please install it using 'codacy-cli install'.") + } + pylintInstallationDirectory := pylint.Info()["installDir"] + pythonRuntime := config.Config.Runtimes()["python"] + pythonBinary := pythonRuntime.Info()["python"] + + log.Printf("Running %s...\n", toolToAnalyze) + if outputFile != "" { + log.Println("Output will be available at", outputFile) + } + tools.RunPylint(workDirectory, pylintInstallationDirectory, pythonBinary, args, autoFix, outputFile) + case "": log.Fatal("You need to specify a tool to run analysis with, e.g., '--tool eslint'", toolToAnalyze) + default: log.Fatal("Trying to run unsupported tool: ", toolToAnalyze) } diff --git a/cmd/init.go b/cmd/init.go index 1356624a..5f06200e 100644 --- a/cmd/init.go +++ b/cmd/init.go @@ -74,18 +74,24 @@ func configFileTemplate(tools []tools.Tool) string { // Default version eslintVersion := "9.3.0" + pylintVersion := "2.13.9" for _, tool := range tools { if tool.Uuid == "f8b29663-2cb2-498d-b923-a10c6a8c05cd" { eslintVersion = tool.Version } + if tool.Uuid == "34225275-f79e-4b85-8126-c7512c987c0d" { + pylintVersion = tool.Version + } } return fmt.Sprintf(`runtimes: - node@22.2.0 + - python@3.9.21 tools: - eslint@%s -`, eslintVersion) + - pylint@%s +`, eslintVersion, pylintVersion) } func buildRepositoryConfigurationFiles(token string) error { diff --git a/cmd/install.go b/cmd/install.go index 284f2c96..310e77a1 100644 --- a/cmd/install.go +++ b/cmd/install.go @@ -35,6 +35,11 @@ func fetchRuntimes(config *cfg.ConfigType) { if err != nil { log.Fatal(err) } + case "python": + err := cfg.InstallPython(r) + if err != nil { + log.Fatal(err) + } default: log.Fatal("Unknown runtime:", r.Name()) } @@ -52,6 +57,12 @@ func fetchTools(config *cfg.ConfigType) { fmt.Println(err.Error()) log.Fatal(err) } + case "pylint": + pythonRuntime := config.Runtimes()["python"] + err := cfg.InstallPylint(pythonRuntime, tool) + if err != nil { + log.Fatal(err) + } default: log.Fatal("Unknown tool:", tool.Name()) } diff --git a/codacy-cli b/codacy-cli new file mode 100755 index 00000000..8c9215f0 Binary files /dev/null and b/codacy-cli differ diff --git a/codacy-cli-local b/codacy-cli-local new file mode 100755 index 00000000..040404c9 Binary files /dev/null and b/codacy-cli-local differ diff --git a/config/pylint-utils.go b/config/pylint-utils.go new file mode 100644 index 00000000..414b6abb --- /dev/null +++ b/config/pylint-utils.go @@ -0,0 +1,58 @@ +package config + +import ( + "fmt" + "log" + "os/exec" + "path" +) + +func getInfoPylint() map[string]string { + pythonRuntime := Config.Runtimes()["python"] + + pythonFolder := fmt.Sprintf("%s@%s", pythonRuntime.Name(), pythonRuntime.Version()) + pythonInstallDir := path.Join(Config.RuntimesDirectory(), pythonFolder, "python") + pylintPath := path.Join(pythonInstallDir, "bin", "pylint") + + return map[string]string{ + "installDir": pythonInstallDir, + "pylint": pylintPath, + } + +} + +// installing in the python runtime because +// f you install Pylint in a different tools folder, it will not work properly because of the following reasons: +// Python Virtual Environment Isolation: +// When you install Pylint in the tools folder (separately from Python), you are essentially mixing environments. +// The python binary located in /Users/yasmin/.cache/codacy/runtimes/python@3.10.16/python/bin/python3 will not have +// access to packages installed elsewhere unless properly referenced. +// PYTHONPATH Limitation: +// You tried passing the tools directory via PYTHONPATH. +// While PYTHONPATH allows modules to be found, it doesn't register packages installed by pip properly. +// Pylint Binary (pylint): +// The pylint binary expects the pylint package to be installed within the same Python environment that is running it. +// If you run: +// /Users/.cache/codacy/runtimes/python@3.10.16/python/bin/python3 -m pylint +// It will look for the pylint module installed under its site-packages directory within: +// /Users/.cache/codacy/runtimes/python@3.10.16/python/lib/python3.10/site-packages +func InstallPylint(pythonRuntime *Runtime, pylint *Runtime) error { + log.Println("Installing Pylint") + + pythonInfo := getInfoPython(pythonRuntime) + + pythonBinary := pythonInfo["python"] + + // to install pylint using python binary + cmd := exec.Command(pythonBinary, "-m", "pip", "install", + fmt.Sprintf("pylint==%s", pylint.Version()), + ) + + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("error installing Pylint: %v\nOutput: %s", err, string(output)) + } + + log.Println("Pylint installed successfully") + return nil +} diff --git a/config/python-utils.go b/config/python-utils.go new file mode 100644 index 00000000..62cc328e --- /dev/null +++ b/config/python-utils.go @@ -0,0 +1,114 @@ +package config + +import ( + "codacy/cli-v2/utils" + "fmt" + "log" + "os" + "path" + "runtime" +) + +func getInfoPython(r *Runtime) map[string]string { + pythonFolder := fmt.Sprintf("%s@%s", r.Name(), r.Version()) + installDir := path.Join(Config.RuntimesDirectory(), pythonFolder) + + var pythonBinary, pipBinary string + + //todo check windows dire, + //had to add python subdir to path since tar extracts it there + if runtime.GOOS == "windows" { + pythonBinary = path.Join(installDir, "Scripts", "python.exe") + pipBinary = path.Join(installDir, "Scripts", "pip.exe") + } else { + pythonBinary = path.Join(installDir, "python", "bin", "python3") + pipBinary = path.Join(installDir, "python", "bin", "pip") + } + + return map[string]string{ + "installDir": installDir, + "python": pythonBinary, + "pip": pipBinary, + } +} + +func getDownloadURL(pythonRuntime *Runtime) string { + + version := pythonRuntime.Version() + goos := runtime.GOOS + goarch := runtime.GOARCH + + var pyArch string + switch goarch { + case "386": + pyArch = "x86" + case "amd64": + pyArch = "x86_64" + case "arm": + pyArch = "armv7l" + case "arm64": + pyArch = "aarch64" + default: + pyArch = goarch + } + + var pyOS string + switch goos { + case "darwin": + pyOS = "apple-darwin" + case "linux": + pyOS = "unknown-linux-gnu" + case "windows": + pyOS = "pc-windows-msvc" + default: + pyOS = goos + } + + releaseVersion := "20250317" + baseURL := "https://github.com/astral-sh/python-build-standalone/releases/download/" + + filename := fmt.Sprintf("cpython-%s+%s-%s-%s-install_only.tar.gz", version, releaseVersion, pyArch, pyOS) + + return fmt.Sprintf("%s%s/%s", baseURL, releaseVersion, filename) + +} + +func InstallPython(r *Runtime) error { + + pythonFolder := fmt.Sprintf("%s@%s", r.Name(), r.Version()) + installDir := path.Join(Config.RuntimesDirectory(), pythonFolder) + log.Println("Fetching python...") + downloadPythonURL := getDownloadURL(r) + pythonTar, err := utils.DownloadFile(downloadPythonURL, Config.RuntimesDirectory()) + if err != nil { + return err + } + + // Make sure the installDir exists + err = os.MkdirAll(installDir, 0777) + if err != nil { + return fmt.Errorf("failed to create install directory: %v", err) + } + + // Open the downloaded file + t, err := os.Open(pythonTar) + defer t.Close() + if err != nil { + return err + } + + // Extract the archive to the desired directory without creating links yet + err = utils.ExtractTarGz(t, installDir) + if err != nil { + return fmt.Errorf("failed to extract archive: %v", err) + } + + //remove tar after extraction + err = os.Remove(pythonTar) + if err != nil { + return fmt.Errorf("failed to delete downloaded archive: %v", err) + } + + log.Println("Python successfully installed at:", installDir) + return nil +} diff --git a/config/runtime.go b/config/runtime.go index 645da86f..536f76ce 100644 --- a/config/runtime.go +++ b/config/runtime.go @@ -32,6 +32,10 @@ func (r *Runtime) populateInfo() { r.info = genInfoNode(r) case "eslint": r.info = genInfoEslint(r) + case "python": + r.info = getInfoPython(r) + case "pylint": + r.info = getInfoPylint() } } diff --git a/tools/pylintRunner.go b/tools/pylintRunner.go new file mode 100644 index 00000000..a2859a44 --- /dev/null +++ b/tools/pylintRunner.go @@ -0,0 +1,84 @@ +package tools + +import ( + "bytes" + "codacy/cli-v2/utils" + "log" + "os" + "os/exec" + "path/filepath" +) + +func RunPylint(repositoryToAnalyseDirectory string, pylintInstallationDirectory string, pythonBinary string, pathsToCheck []string, autoFix bool, outputFile string) { + + // Prepare the command to run pylint as a module with JSON output + var args []string + args = append(args, "-m", "pylint", "--output-format=json") + + // Create a temporary file for JSON output if an output file is specified + tempFile := "" + if outputFile != "" { + tempFile = filepath.Join(os.TempDir(), "pylint_output.json") + args = append(args, "--output", tempFile) + } + + // Add files/directories to check + if len(pathsToCheck) > 0 { + args = append(args, pathsToCheck...) + } else { + args = append(args, repositoryToAnalyseDirectory) + } + + cmd := exec.Command(pythonBinary, args...) + cmd.Dir = repositoryToAnalyseDirectory + + // Set stderr and stdout to be displayed + cmd.Stderr = os.Stderr + + // For terminal output capture mode + var stdout bytes.Buffer + if outputFile == "" { + // Terminal output mode - capture JSON and print SARIF directly + cmd.Stdout = &stdout + } else { + // File output mode - show output in terminal + cmd.Stdout = os.Stdout + } + + // Run pylint + log.Printf("Running pylint command: %v", cmd.Args) + // Pylint returns non-zero exit codes when it finds issues, so we're not checking the error + cmd.Run() + + if outputFile != "" { + // Read the JSON output from the temporary file + outputData, err := os.ReadFile(tempFile) + if err != nil { + log.Printf("Failed to read pylint output from %s: %v", tempFile, err) + return + } + + // Delete temporary file + defer os.Remove(tempFile) + + // Convert JSON to SARIF using the utility function + sarifData := utils.ConvertPylintToSarif(outputData) + + // Write SARIF to the output file + err = os.WriteFile(outputFile, sarifData, 0644) + if err != nil { + log.Printf("Failed to write SARIF output to %s: %v", outputFile, err) + } + + log.Printf("SARIF output saved to: %s\n", outputFile) + } else { + // Get the JSON output from the buffer + jsonOutput := stdout.Bytes() + + // Convert JSON to SARIF + sarifOutput := utils.ConvertPylintToSarif(jsonOutput) + + // Print the SARIF output to stdout + os.Stdout.Write(sarifOutput) + } +} diff --git a/utils/extract.go b/utils/extract.go index 5e4b21c2..0f105728 100644 --- a/utils/extract.go +++ b/utils/extract.go @@ -2,6 +2,7 @@ package utils import ( "context" + "fmt" "github.com/mholt/archiver/v4" "io" "os" @@ -14,36 +15,39 @@ func ExtractTarGz(archive *os.File, targetDir string) error { Archival: archiver.Tar{}, } + var symlinks []archiver.File // Collect symlinks for second pass + handler := func(ctx context.Context, f archiver.File) error { path := filepath.Join(targetDir, f.NameInArchive) switch f.IsDir() { case true: - // create a directory - //fmt.Println("creating: " + f.NameInArchive) + // Create directory if it doesn't exist err := os.MkdirAll(path, 0777) if err != nil { return err } case false: - //log.Print("extracting: " + f.NameInArchive) - - // if is a symlink + // If it's a symlink, defer its creation if f.LinkTarget != "" { - os.Remove(path) - err := os.Symlink(f.LinkTarget, path) - if err != nil { - return err - } - return nil + symlinks = append(symlinks, f) + return nil // Skip creating the symlink for now } - // write a file + // Ensure the parent directory exists + parentDir := filepath.Dir(path) + err := os.MkdirAll(parentDir, 0777) + if err != nil { + return fmt.Errorf("failed to create parent directory: %v", err) + } + + // Create file with original permissions w, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, f.Mode()) if err != nil { - return err + return fmt.Errorf("failed to create file: %v", err) } + defer w.Close() stream, _ := f.Open() defer stream.Close() @@ -52,15 +56,30 @@ func ExtractTarGz(archive *os.File, targetDir string) error { if err != nil { return err } - w.Close() } return nil } + // First Pass: Extract all files and directories (no symlinks yet) err := format.Extract(context.Background(), archive, nil, handler) if err != nil { return err } + + // Second Pass: Create symlinks + for _, f := range symlinks { + path := filepath.Join(targetDir, f.NameInArchive) + + // Remove existing file if any + os.Remove(path) + + // Create the symlink + err := os.Symlink(f.LinkTarget, path) + if err != nil { + fmt.Printf("Failed to create symlink: %s -> %s, Error: %v\n", path, f.LinkTarget, err) + } + } + return nil } diff --git a/utils/pylint_sarif_converter.go b/utils/pylint_sarif_converter.go new file mode 100644 index 00000000..2371e7e3 --- /dev/null +++ b/utils/pylint_sarif_converter.go @@ -0,0 +1,149 @@ +package utils + +import ( + "encoding/json" + "log" +) + +// ConvertPylintToSarif converts pylint JSON output to SARIF format +func ConvertPylintToSarif(jsonData []byte) []byte { + var pylintResults []map[string]interface{} + err := json.Unmarshal(jsonData, &pylintResults) + if err != nil { + log.Printf("Failed to parse pylint JSON output: %v", err) + return createEmptySarif() + } + + // Create SARIF structure + sarif := map[string]interface{}{ + "version": "2.1.0", + "$schema": "http://json.schemastore.org/sarif-2.1.0-rtm.5", + "runs": []map[string]interface{}{ + { + "tool": map[string]interface{}{ + "driver": map[string]interface{}{ + "name": "Pylint", + "informationUri": "https://pylint.org", + "rules": []map[string]interface{}{}, + }, + }, + "artifacts": []map[string]interface{}{}, + "results": []map[string]interface{}{}, + }, + }, + } + + // Process pylint results + run := sarif["runs"].([]map[string]interface{})[0] + rules := map[string]bool{} + artifacts := map[string]int{} + + // Track files for artifacts + for _, result := range pylintResults { + path, ok := result["path"].(string) + if ok && path != "" { + artifacts[path] = 0 + } + } + + // Create artifact entries + artifactsList := []map[string]interface{}{} + i := 0 + for path := range artifacts { + artifactsList = append(artifactsList, map[string]interface{}{ + "location": map[string]interface{}{ + "uri": path, + }, + }) + artifacts[path] = i + i++ + } + run["artifacts"] = artifactsList + + // Process results + results := []map[string]interface{}{} + for _, pylintResult := range pylintResults { + messageId, messageIdOk := pylintResult["message-id"].(string) + symbol, symbolOk := pylintResult["symbol"].(string) + message, messageOk := pylintResult["message"].(string) + path, pathOk := pylintResult["path"].(string) + line, lineOk := pylintResult["line"].(float64) + column, columnOk := pylintResult["column"].(float64) + + if !messageIdOk || !symbolOk || !messageOk || !pathOk || !lineOk { + continue + } + + // Add rule if not already added + ruleId := symbol + if !rules[ruleId] { + rules[ruleId] = true + rule := map[string]interface{}{ + "id": ruleId, + "shortDescription": map[string]interface{}{ + "text": messageId, + }, + } + run["tool"].(map[string]interface{})["driver"].(map[string]interface{})["rules"] = + append(run["tool"].(map[string]interface{})["driver"].(map[string]interface{})["rules"].([]map[string]interface{}), rule) + } + + // Create result + result := map[string]interface{}{ + "ruleId": ruleId, + "message": map[string]interface{}{ + "text": message, + }, + "locations": []map[string]interface{}{ + { + "physicalLocation": map[string]interface{}{ + "artifactLocation": map[string]interface{}{ + "uri": path, + "index": artifacts[path], + }, + "region": map[string]interface{}{ + "startLine": int(line), + "startColumn": 1, // Default to 1 if column is not available + }, + }, + }, + }, + } + + // Use column if available + if columnOk { + result["locations"].([]map[string]interface{})[0]["physicalLocation"].(map[string]interface{})["region"].(map[string]interface{})["startColumn"] = int(column) + } + + results = append(results, result) + } + run["results"] = results + + // Convert to JSON + sarifData, err := json.MarshalIndent(sarif, "", " ") + if err != nil { + log.Printf("Failed to marshal SARIF data: %v", err) + return createEmptySarif() + } + + return sarifData +} + +// createEmptySarif returns an empty SARIF structure as a byte array +func createEmptySarif() []byte { + return []byte(`{ + "version": "2.1.0", + "$schema": "http://json.schemastore.org/sarif-2.1.0-rtm.5", + "runs": [ + { + "tool": { + "driver": { + "name": "Pylint", + "informationUri": "https://pylint.org" + } + }, + "results": [] + } + ] + }`) +}