Allow url accessible rules (#2848)

Author: niclim

projects/optic/src/__tests__/integration/__snapshots__/diff.test.ts.snap

@@ -533,8 +533,7 @@ exports[`diff with mock server custom rules 1`] = `
 `;
 
 exports[`diff with mock server extends 1`] = `
-"Extending ruleset from @test-org/ruleset-id
-x Empty example-api-v0.json
+"x Empty example-api-v0.json
 Operations: 1 operation added, 3 changed, 1 removed
 x  Checks: 3/5 passed 
 

projects/optic/src/commands/diff/generate-rule-runner.ts

@@ -1,5 +1,5 @@
 import path from 'path';
-import { ConfigRuleset, OpticCliConfig } from '../../config';
+import { ConfigRuleset, OpticCliConfig, initializeRules } from '../../config';
 import { StandardRulesets } from '@useoptic/standard-rulesets';
 import {
   RuleRunner,
@@ -19,6 +19,16 @@ export function setRulesets(rulesets: (Ruleset | ExternalRule)[]) {
 }
 
 const isLocalJsFile = (name: string) => name.endsWith('.js');
+const isUrl = (name: string) => {
+  try {
+    const parsed = new URL(name);
+    if (parsed.protocol === 'http:' || parsed.protocol === 'https:')
+      return true;
+    return false;
+  } catch (e) {
+    return false;
+  }
+};
 
 type InputPayload = Parameters<typeof prepareRulesets>[0];
 
@@ -35,19 +45,28 @@ const getStandardToUse = async (options: {
     );
     return config.ruleset;
   } else if (options.specRuleset) {
-    try {
-      const ruleset = await options.config.client.getStandard(
-        options.specRuleset
-      );
-      return ruleset.config.ruleset;
-    } catch (e) {
-      logger.warn(
-        `${chalk.red('Warning:')} Could not download standard ${
+    if (options.specRuleset.startsWith('@')) {
+      try {
+        const ruleset = await options.config.client.getStandard(
           options.specRuleset
-        }. Please check the ruleset name and whether you are authenticated (run: optic login).`
-      );
-      process.exitCode = 1;
-      return [];
+        );
+        return ruleset.config.ruleset;
+      } catch (e) {
+        logger.warn(
+          `${chalk.red('Warning:')} Could not download standard ${
+            options.specRuleset
+          }. Please check the ruleset name and whether you are authenticated (run: optic login).`
+        );
+        process.exitCode = 1;
+        return [];
+      }
+    } else {
+      const rules: any = {
+        extends: options.specRuleset,
+      };
+      await initializeRules(rules, options.config.client);
+
+      return rules.ruleset;
     }
   } else {
     return options.config.ruleset;
@@ -105,6 +124,12 @@ export const generateRuleRunner = async (
           ? path.dirname(options.config.configPath)
           : process.cwd();
         localRulesets[rule.name] = path.resolve(rootPath, rule.name); // the path is the name
+      } else if (isUrl(rule.name)) {
+        hostedRulesets[rule.name] = {
+          uploaded_at: String(Math.random()),
+          url: rule.name,
+          should_decompress: false,
+        };
       } else {
         rulesToFetch.push(rule.name);
       }
@@ -118,6 +143,7 @@ export const generateRuleRunner = async (
         hostedRulesets[hostedRuleset.name] = {
           uploaded_at: hostedRuleset.uploaded_at,
           url: hostedRuleset.url,
+          should_decompress: true,
         };
       }
     }

projects/optic/src/config.ts

@@ -1,4 +1,5 @@
 import fs from 'node:fs/promises';
+import fetch from 'node-fetch';
 import yaml from 'js-yaml';
 import { UserError, isTruthyStringValue } from '@useoptic/openapi-utilities';
 import Ajv from 'ajv';
@@ -217,14 +218,28 @@ export const initializeRules = async (
   client: OpticBackendClient
 ) => {
   let rulesetMap: Map<string, ConfigRuleset> = new Map();
+  let rawRulesets = config.ruleset ? config.ruleset : [];
   if (config.extends) {
-    console.log(`Extending ruleset from ${config.extends}`);
+    logger.debug(`Extending ruleset from ${config.extends}`);
 
     try {
-      const response = await client.getStandard(config.extends);
-      rulesetMap = new Map(
-        response.config.ruleset.map((conf) => [conf.name, conf])
-      );
+      if (config.extends.startsWith('@')) {
+        const response = await client.getStandard(config.extends);
+        rulesetMap = new Map(
+          response.config.ruleset.map((conf) => [conf.name, conf])
+        );
+      } else {
+        // Assumption is that we're fetching a yaml file
+        const response = await fetch(config.extends).then((response) => {
+          if (response.status !== 200) {
+            throw new Error(`received status code ${response.status}`);
+          } else {
+            return response.text();
+          }
+        });
+        const parsed = yaml.load(response);
+        rawRulesets.push(...(parsed as any).ruleset);
+      }
     } catch (e) {
       console.error(e);
       console.log(
@@ -233,8 +248,8 @@ export const initializeRules = async (
     }
   }
 
-  if (config.ruleset) {
-    for (const ruleset of config.ruleset) {
+  if (rawRulesets.length) {
+    for (const ruleset of rawRulesets) {
       if (typeof ruleset === 'string') {
         rulesetMap.set(ruleset, { name: ruleset, config: {} });
       } else if (typeof ruleset === 'object' && ruleset !== null) {

projects/rulesets-base/src/custom-rulesets/__tests__/download-ruleset.test.ts

@@ -20,7 +20,8 @@ describe('downloadRuleset', () => {
       downloadRuleset(
         'test-ruleset',
         'https://some-url.com',
-        '2022-11-01T19:32:22.148Z'
+        '2022-11-01T19:32:22.148Z',
+        true
       )
     ).rejects.toThrow(new Error('Downloading ruleset failed (404): Missing'));
   });
@@ -35,7 +36,8 @@ describe('downloadRuleset', () => {
     await downloadRuleset(
       'test-ruleset',
       'https://some-url.com',
-      '2022-11-01T19:32:22.148Z'
+      '2022-11-01T19:32:22.148Z',
+      true
     );
     expect(fs.mkdir).toBeCalled();
     expect(fs.writeFile).toHaveBeenCalledWith(
@@ -49,7 +51,8 @@ describe('downloadRuleset', () => {
     await downloadRuleset(
       'test-ruleset',
       'https://some-url.com',
-      '2022-11-01T19:32:22.148Z'
+      '2022-11-01T19:32:22.148Z',
+      true
     );
 
     expect(fetch).not.toBeCalled();

projects/rulesets-base/src/custom-rulesets/__tests__/prepare-ruleset.test.ts

@@ -32,6 +32,7 @@ describe('prepareRulesets', () => {
           '@team/custom-ruleset': {
             url: 'https://some-url.com',
             uploaded_at: '123',
+            should_decompress: true,
           },
         },
         standardRulesets: {
@@ -76,6 +77,7 @@ describe('prepareRulesets', () => {
         '@team/custom-ruleset': {
           url: 'https://some-url.com',
           uploaded_at: '123',
+          should_decompress: true,
         },
       },
       standardRulesets: {

projects/rulesets-base/src/custom-rulesets/download-ruleset.ts

@@ -7,7 +7,8 @@ import path from 'node:path';
 export async function downloadRuleset(
   name: string,
   url: string,
-  uploaded_at: string
+  uploaded_at: string,
+  should_decompress: boolean
 ): Promise<string> {
   const filepath = path.join(os.tmpdir(), name, `${uploaded_at}.js`);
   try {
@@ -22,14 +23,18 @@ export async function downloadRuleset(
       `Downloading ruleset failed (${resp.status}): ${await resp.text()}`
     );
   }
-
-  const compressed = await resp.buffer();
-  const decompressed = zlib.brotliDecompressSync(compressed);
+  let raw: Buffer;
+  if (should_decompress) {
+    const compressed = await resp.buffer();
+    raw = zlib.brotliDecompressSync(compressed);
+  } else {
+    raw = await resp.buffer();
+  }
 
   const filefolder = path.dirname(filepath);
   // Does not error if folder exists when recursive = true
   await fs.mkdir(filefolder, { recursive: true });
-  await fs.writeFile(filepath, decompressed);
+  await fs.writeFile(filepath, raw);
 
   return filepath;
 }

projects/rulesets-base/src/custom-rulesets/prepare-rulesets.ts

@@ -15,6 +15,7 @@ export type RulesetPayload = {
     {
       uploaded_at: string;
       url: string;
+      should_decompress: boolean;
     }
   >;
   standardRulesets: Record<
@@ -81,7 +82,8 @@ export async function prepareRulesets(
         rulesetPath = await downloadRuleset(
           ruleset.name,
           hostedRuleset.url,
-          hostedRuleset.uploaded_at
+          hostedRuleset.uploaded_at,
+          hostedRuleset.should_decompress
         );
       } catch (e) {
         warnings.push(`Loading ruleset ${ruleset.name} failed`);