@W-18706176: Changes to download latest release version of the API (#231)

* Changes to download latest release version of the API

Author: unandyala

.github/workflows/test.yml

@@ -26,6 +26,10 @@ jobs:
         with:
           path: "**/node_modules"
           key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+      - name: Install oasdiff
+        run: |
+          curl -fsSL https://raw.githubusercontent.com/oasdiff/oasdiff/main/install.sh | sh
+          oasdiff --version
       - run: npm ci
         if: ${{ steps.cache-nodemodules.outputs.cache-hit != 'true' }}
       - run: npm run compile

src/download/downloadCommand.test.ts

@@ -10,14 +10,17 @@ import tmp from "tmp";
 import chai from "chai";
 import chaiFs from "chai-fs";
 import { DownloadCommand } from "./downloadCommand";
+import _ from "lodash";
 
 chai.use(chaiFs);
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const assetSearchResults = require("../../testResources/download/resources/assetSearch.json");
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
-const asset = require("../../testResources/download/resources/getAsset");
+const assetResource = require("../../testResources/download/resources/getAsset");
+// Create a local copy to avoid mutating shared test resources that other tests depend on
+const asset = _.cloneDeep(assetResource);
 // Use a shorter URL for better readability
 asset.files.find((file) => file.classifier === "fat-raml").externalLink =
   "https://short.url/raml.zip";
@@ -66,7 +69,11 @@ function setup({
       // Intercept searchExchange request
       .nock("https://anypoint.mulesoft.com/exchange/api/v2", (scope) =>
         scope
-          .get(`/assets?search=${encodeURIComponent(search)}&types=rest-api`)
+          .get(
+            `/assets?search=${encodeURIComponent(
+              search
+            )}&types=rest-api&limit=50&offset=0`
+          )
           .reply(200, [assetSearchResults[0]])
       )
       // Intercept search requests

src/download/exchangeDirectoryParser.test.ts

@@ -1,11 +1,11 @@
 /*
- * Copyright (c) 2020, salesforce.com, inc.
+ * Copyright (c) 2025, salesforce.com, inc.
  * All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
 
-import { extractFiles } from "./exchangeDirectoryParser";
+import { extractFiles, extractFile } from "./exchangeDirectoryParser";
 
 import { expect, default as chai } from "chai";
 import chaiAsPromised from "chai-as-promised";
@@ -33,6 +33,33 @@ before(() => {
   chai.use(chaiAsPromised);
 });
 
+describe("extractFile", () => {
+  it("should reject with an error message when trying to extract an invalid zip file", async () => {
+    const directory = tmp.dirSync();
+    const invalidZipPath = path.join(directory.name, "invalid.zip");
+
+    // Create a file that looks like a zip but isn't valid
+    fs.writeFileSync(invalidZipPath, "This is not a valid zip file");
+
+    await expect(extractFile(invalidZipPath)).to.be.rejectedWith(
+      `Failed to extract ${invalidZipPath}, probably not a zip file`
+    );
+  });
+
+  it("should successfully extract a valid zip file", async () => {
+    const directory = tmp.dirSync();
+    const zipPath = path.join(directory.name, "api1.zip");
+
+    // Create a valid zip file
+    await createZipFile(directory, "api1.zip");
+
+    const extractedPath = await extractFile(zipPath);
+
+    expect(fs.existsSync(extractedPath)).to.be.true;
+    expect(fs.existsSync(path.join(extractedPath, "exchange.json"))).to.be.true;
+  });
+});
+
 describe("extractFiles", () => {
   it("should extract a zip file into the specified directory", async () => {
     const directory = tmp.dirSync();

src/download/exchangeDownloader.test.ts

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, salesforce.com, inc.
+ * Copyright (c) 2025, salesforce.com, inc.
  * All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
@@ -176,7 +176,7 @@ describe("exchangeDownloader", () => {
   describe("searchExchange", () => {
     it("can download multiple files", async () => {
       nock("https://anypoint.mulesoft.com/exchange/api/v2")
-        .get("/assets?search=searchString&types=rest-api")
+        .get("/assets?search=searchString&types=rest-api&limit=50&offset=0")
         .reply(200, assetSearchResults);
 
       return searchExchange("AUTH_TOKEN", "searchString").then((res) => {
@@ -335,10 +335,15 @@ describe("exchangeDownloader", () => {
           Authorization: "Bearer AUTH_TOKEN",
         },
       })
-        .get("/assets?search=searchString&types=rest-api")
+        .get("/assets?search=searchString&types=rest-api&limit=50&offset=0")
         .reply(200, [assetSearchResults[0]]);
     });
 
+    afterEach(() => {
+      // Clean up any remaining nock interceptors for this describe block
+      nock.cleanAll();
+    });
+
     it("searches Exchange and filters by deployment", () => {
       scope
         .get("/shop-products-categories-api-v1")
@@ -390,7 +395,7 @@ describe("exchangeDownloader", () => {
       asset.fatRaml = {
         classifier: "fat-raml",
         packaging: "zip",
-        externalLink: "https://short.url/raml.zip",
+        externalLink: "https://short.url/test",
         createdDate: "2020-02-05T21:26:01.199Z",
         md5: "87b3ad2b2aa17639b52f0cc83c5a8d40",
         sha1: "f2b9b2de50b7250616e2eea8843735b57235c22b",

src/download/exchangeDownloader.ts

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, salesforce.com, inc.
+ * Copyright (c) 2025, salesforce.com, inc.
  * All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
@@ -30,6 +30,8 @@ const ANYPOINT_API_URI_V2 = `${ANYPOINT_BASE_URI}/api/v2`;
 const DEPLOYMENT_DEPRECATION_WARNING =
   "The 'deployment' argument is deprecated. The latest RAML specification that is published to Anypoint Exchange will be downloaded always.";
 
+// Only allows MAJOR.MINOR.PATCH (no suffixes). see https://semver.org/
+const releaseSemverRegex = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/;
 /**
  * Makes an HTTP call to the url with the options passed. If the calls due to
  * a 5xx, 408, 420 or 429, it retries the call with the retry options passed
@@ -73,7 +75,15 @@ export async function downloadRestApi(
   }
   try {
     await fs.ensureDir(destinationFolder);
-    const zipFilePath = path.join(destinationFolder, `${restApi.assetId}.zip`);
+    let zipFilePath = path.join(destinationFolder, `${restApi.assetId}.zip`);
+
+    if (isOas) {
+      //For OAS, download clean latest versions from multiple version groups
+      zipFilePath = path.join(
+        destinationFolder,
+        `${restApi.assetId}-${restApi.version}.zip`
+      );
+    }
 
     const fatRaml = restApi.fatRaml;
     const fatOas = restApi.fatOas;
@@ -104,6 +114,7 @@ export async function downloadRestApi(
  * Download the API specifications
  * @param restApi - Metadata of the API
  * @param destinationFolder - Destination directory for the download
+ * @param isOas - True for Open Api Specification
  */
 export async function downloadRestApis(
   restApi: RestApi[],
@@ -204,8 +215,9 @@ export async function searchExchange(
   accessToken: string,
   searchString: string
 ): Promise<RestApi[]> {
+  //TODO: We may have to handle pagination in the future if the number of APIs returned is more than 50
   return runFetch(
-    `${ANYPOINT_API_URI_V2}/assets?search=${searchString}&types=rest-api`,
+    `${ANYPOINT_API_URI_V2}/assets?search=${searchString}&types=rest-api&limit=50&offset=0`,
     {
       headers: {
         Authorization: `Bearer ${accessToken}`,
@@ -299,11 +311,13 @@ export async function getSpecificApi(
  * @param query - Exchange search query
  * @param [deployment] - RegExp matching the desired deployment targets
  *
+ * @param isOas - True to get Open API Specifications, false for RAML
  * @returns Information about the APIs found.
  */
 export async function search(
   query: string,
-  deployment?: RegExp
+  deployment?: RegExp,
+  isOas = false
 ): Promise<RestApi[]> {
   if (deployment) {
     ramlToolLogger.warn(DEPLOYMENT_DEPRECATION_WARNING);
@@ -314,6 +328,9 @@ export async function search(
     process.env.ANYPOINT_PASSWORD
   );
   const apis = await searchExchange(token, query);
+  if (isOas) {
+    return getLatestCleanApis(apis, token);
+  }
   const promises = apis.map(async (api) => {
     const version = await getVersionByDeployment(token, api, deployment);
     return version
@@ -322,3 +339,108 @@ export async function search(
   });
   return Promise.all(promises);
 }
+
+/**
+ * Gets information about all the APIs from exchange that match the given search
+ * string.
+ * If it fails to get information about the deployed version of an API, it
+ * removes all the version specific information from the returned object.
+ *
+ * @param apis - Array of apis to get the latest versions
+ * @param {string} accessToken
+ *
+ * @returns Information about the APIs found.
+ */
+export async function getLatestCleanApis(
+  apis: RestApi[],
+  accessToken: string
+): Promise<RestApi[]> {
+  // Get all API versions in parallel
+  const apiVersionPromises = apis.map(async (api) => {
+    const versions = await getLatestCleanApiVersions(accessToken, api);
+    if (!versions || versions.length === 0) {
+      return { api, versions: [] };
+    }
+    return { api, versions };
+  });
+
+  const allApiVersions = await Promise.all(apiVersionPromises);
+  // Create promises for all API versions and process them in parallel
+  const promises = [];
+  for (const { api, versions } of allApiVersions) {
+    for (const version of versions) {
+      promises.push(
+        getSpecificApi(accessToken, api.groupId, api.assetId, version)
+      );
+    }
+  }
+  return Promise.all(promises);
+}
+
+/**
+ * @description Returns the latest clean (MAJOR.MINOR.PATCH) API versions from multiple version groups (V1, V2..) of an API
+ *
+ * @export
+ * @param {string} accessToken
+ * @param {RestApi} restApi
+ * @returns {Promise<string>} Returned the version string from the instance fetched asset.version value
+ */
+export async function getLatestCleanApiVersions(
+  accessToken: string,
+  restApi: RestApi
+): Promise<void | string[]> {
+  const logPrefix = "[exchangeDownloader][getLatestCleanApiVersions]";
+
+  let asset: void | RawRestApi;
+  try {
+    asset = await getAsset(
+      accessToken,
+      `${restApi.groupId}/${restApi.assetId}`
+    );
+  } catch (error) {
+    ramlToolLogger.error(`${logPrefix} Error fetching asset:`, error);
+    return;
+  }
+
+  if (!asset) {
+    ramlToolLogger.log(
+      `${logPrefix} No asset found for ${restApi.assetId}, returning`
+    );
+    return;
+  }
+
+  if (!asset.versionGroups) {
+    ramlToolLogger.error(
+      `${logPrefix} The rest API ${restApi.assetId} is missing asset.versionGroups`
+    );
+    return;
+  }
+  const versions: string[] = [];
+  asset.versionGroups.forEach((versionGroup) => {
+    const version = getLatestReleaseVersion(versionGroup);
+    if (version) {
+      versions.push(version);
+    }
+  });
+  return versions;
+}
+
+function getLatestReleaseVersion(versionGroup: {
+  versions: Array<{ version: string }>;
+}): void | string {
+  if (!versionGroup.versions || versionGroup.versions.length === 0) {
+    return;
+  }
+  const releaseAssetVersions = versionGroup.versions.filter((version) => {
+    return releaseSemverRegex.test(version.version);
+  });
+  // Sort versions and get the latest
+  return releaseAssetVersions.sort((instanceA, instanceB) => {
+    const [aMajor, aMinor, aPatch] = instanceA.version.split(".").map(Number);
+    const [bMajor, bMinor, bPatch] = instanceB.version.split(".").map(Number);
+
+    if (aMajor !== bMajor) return bMajor - aMajor;
+    if (aMinor !== bMinor) return bMinor - aMinor;
+    return bPatch - aPatch;
+  })[0].version;
+}