Feat: Reading usns from filepath (#1303)

* fix: Adding functionality for reading usns from a file

* fix: action is being read from a Dockerfile

* fix: removing extra parameter

Author: pacostas

actions/stack/get-usns/README.md

@@ -16,6 +16,10 @@ inputs:
   last_usns_filepath:
     description: 'Similar to last_usns, but instead of pointing into a variable, it points to a file'
     required: false
+  api_url:
+    description: 'URL of the Ubuntu security notices JSON API (https or file:// for a local JSON file)'
+    required: false
+    default: 'https://ubuntu.com/security/notices.json'
   packages:
     description: 'JSON array of stack package names'
     required: false
@@ -29,15 +33,21 @@ inputs:
   usns_output_path:
     description: 'Path to output usns JSON file'
     required: false
+  pages:
+    description: 'Number of pages to fetch from the API (each page has 20 notices)'
+    required: false
+    default: '1'
 
 runs:
   using: 'docker'
-  image: 'docker://ghcr.io/paketo-buildpacks/actions/stack/get-usns:latest'
+  image: 'Dockerfile'
   args:
   - "--last-usns"
   - "${{ inputs.last_usns }}"
   - "--last-usns-filepath"
   - "${{ inputs.last_usns_filepath }}"
+  - "--api-url"
+  - "${{ inputs.api_url }}"
   - "--packages"
   - "${{ inputs.packages }}"
   - "--packages-filepath"
@@ -46,3 +56,5 @@ runs:
   - "${{ inputs.distribution }}"
   - "--output"
   - "${{ inputs.usns_output_path }}"
+  - "--pages"
+  - "${{ inputs.pages }}"

actions/stack/get-usns/action.yml

@@ -7,9 +7,11 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
 	"slices"
+	"strings"
 	"time"
 )
 
@@ -47,13 +49,14 @@ type PatchedUsnsInputOutput struct {
 
 func main() {
 	var config struct {
+		APIUrl               string
 		Distro               string
 		LastUSNsJSON         string
 		LastUSNsJSONFilepath string
 		Output               string
 		PackagesJSON         string
 		PackagesJSONFilepath string
-		APIUrl               string
+		Pages                int
 	}
 
 	flag.StringVar(&config.LastUSNsJSON,
@@ -67,7 +70,7 @@ func main() {
 	flag.StringVar(&config.APIUrl,
 		"api-url",
 		JSON_API_URL,
-		"URL of the Ubuntu security notices JSON API")
+		"URL of the Ubuntu security notices JSON API (https or file:// for a local JSON file)")
 	flag.StringVar(&config.PackagesJSON,
 		"packages",
 		"",
@@ -84,12 +87,19 @@ func main() {
 		"output",
 		"",
 		"Path to output JSON file")
+	flag.IntVar(&config.Pages,
+		"pages",
+		1,
+		"Number of pages to fetch from the API (default: 1)")
 
 	flag.Parse()
 
 	if !slices.Contains(supportedDistros, config.Distro) {
 		log.Fatalf("--distro flag has to be one of the following values: %v", supportedDistros)
 	}
+	if config.Pages < 1 {
+		log.Fatalf("--pages must be at least 1, got %d", config.Pages)
+	}
 
 	lastPatchedUSNs := []PatchedUsnsInputOutput{}
 	if config.LastUSNsJSON != "" {
@@ -132,9 +142,23 @@ func main() {
 		}
 	}
 
-	newUSNs, err := getNewUSNsFromJSONApi(config.APIUrl, lastPatchedUSNs, config.Distro)
-	if err != nil {
-		log.Fatal(err)
+	var newUSNs []USN
+	var err error
+	if path, ok := fileURLPath(config.APIUrl); ok {
+		newUSNs, err = getNewUSNsFromFilepath(path)
+		if err != nil {
+			log.Fatal(err)
+		}
+	} else {
+		newUSNs, err = getNewUSNsFromJSONApi(config.APIUrl, lastPatchedUSNs, config.Distro, config.Pages)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	fmt.Println("Recent USNs found:")
+	for _, usn := range newUSNs {
+		fmt.Printf("%s with name %s\n", usn.ID, usn.Title)
 	}
 
 	filteredUSNs := filterUSNsByPackages(newUSNs, packages, config.Distro)
@@ -225,20 +249,48 @@ func transformUSNsForOutput(usns []USN, distro string) []PatchedUsnsInputOutput
 	return output
 }
 
-func getNewUSNsFromJSONApi(jsonApiUrl string, lastPatchedUSNs []PatchedUsnsInputOutput, distro string) ([]USN, error) {
+func fileURLPath(rawURL string) (string, bool) {
+	u, err := url.Parse(strings.TrimSpace(rawURL))
+	if err != nil || u.Scheme != "file" {
+		return "", false
+	}
+	path := u.Path
+	if u.Host != "" {
+		path = u.Host + path
+	}
+	return path, true
+}
+
+func getNewUSNsFromFilepath(path string) ([]USN, error) {
+	fileContent, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	var data struct {
+		Notices []USN `json:"notices"`
+	}
+	if err = json.Unmarshal(fileContent, &data); err != nil {
+		return nil, err
+	}
+	return data.Notices, nil
+}
+
+const pageSize = 20
+
+func getNewUSNsFromJSONApi(jsonApiUrl string, lastPatchedUSNs []PatchedUsnsInputOutput, distro string, numPages int) ([]USN, error) {
 	var allUSNs []USN
 
-	offsets := []int{0, 20}
-	for _, offset := range offsets {
-		paginatedUrl := fmt.Sprintf("%s?release=%s&limit=%d&offset=%d", jsonApiUrl, distro, 20, offset)
+	for page := 0; page < numPages; page++ {
+		offset := page * pageSize
+		paginatedUrl := fmt.Sprintf("%s?release=%s&limit=%d&offset=%d", jsonApiUrl, distro, pageSize, offset)
 		usns, err := fetchUSNPage(paginatedUrl)
 		if err != nil {
 			return nil, err
 		}
 		allUSNs = append(allUSNs, usns...)
 	}
 
-	fmt.Println("Looking for new USNs...")
 	var newUSNs []USN
 	for _, usn := range allUSNs {
 
@@ -254,14 +306,6 @@ func getNewUSNsFromJSONApi(jsonApiUrl string, lastPatchedUSNs []PatchedUsnsInput
 		})
 	}
 
-	if len(newUSNs) > 0 {
-		for _, usn := range newUSNs {
-			fmt.Printf("New USN found: %s with name %s\n", usn.ID, usn.Title)
-		}
-	} else {
-		fmt.Println("No new USNs found")
-	}
-
 	return newUSNs, nil
 }
 

actions/stack/get-usns/entrypoint/main.go

@@ -87,7 +87,38 @@ func TestEntrypoint(t *testing.T) {
 				Expect(err).NotTo(HaveOccurred())
 
 				Eventually(session).Should(gexec.Exit(0), func() string { return fmt.Sprintf("output -> \n%s\n", buffer.Contents()) })
-				Expect(string(buffer.Contents())).To(ContainSubstring("New USN found:"))
+				Expect(string(buffer.Contents())).To(ContainSubstring("Recent USNs found:"))
+
+				Expect(outputFilepath).To(BeARegularFile())
+				contents, err := os.ReadFile(outputFilepath)
+				Expect(err).NotTo(HaveOccurred())
+
+				Expect(string(contents)).To(ContainSubstring(`"id":"USN-7967-1"`))
+				Expect(string(contents)).To(ContainSubstring(`"avahi"`))
+			})
+		})
+
+		context("Fetches usns from a file (file://) and previous usns are empty", func() {
+			it("outputs the correct patched usns", func() {
+				testdataPath, err := filepath.Abs("testdata/notices0-20.json")
+				Expect(err).NotTo(HaveOccurred())
+				apiURL := "file://" + testdataPath
+
+				command := exec.Command(
+					entrypoint,
+					"--api-url", apiURL,
+					"--packages", `["avahi", "simgear"]`,
+					"--distro", "noble",
+					"--output", outputFilepath,
+				)
+
+				buffer := gbytes.NewBuffer()
+
+				session, err := gexec.Start(command, buffer, buffer)
+				Expect(err).NotTo(HaveOccurred())
+
+				Eventually(session).Should(gexec.Exit(0), func() string { return fmt.Sprintf("output -> \n%s\n", buffer.Contents()) })
+				Expect(string(buffer.Contents())).To(ContainSubstring("Recent USNs found:"))
 
 				Expect(outputFilepath).To(BeARegularFile())
 				contents, err := os.ReadFile(outputFilepath)
@@ -156,6 +187,7 @@ func TestEntrypoint(t *testing.T) {
 					"--distro", "jammy",
 					"--last-usns-filepath", "testdata/amd64-jammy-tiny-last-patched-usns.json",
 					"--output", outputFilepath,
+					"--pages", "2",
 				)
 
 				buffer := gbytes.NewBuffer()