feat: add proper handling for additionalProperties

Author: AlbinaBlazhko17

packages/core/src/rules/oas3/no-illogical-composition-keywords.ts

@@ -249,7 +249,19 @@ function arePropertiesMutuallyExclusive(
   return { isExclusive: false };
 }
 
-function areSignaturesMutuallyExclusive(sig1: SchemaSignature, sig2: SchemaSignature): ReturnType {
+function areSignaturesMutuallyExclusive(
+  sig1: SchemaSignature,
+  sig2: SchemaSignature,
+  depth: number = 0
+): ReturnType {
+  const MAX_DEPTH = 10;
+  if (depth > MAX_DEPTH) {
+    // If we've gone too deep, assume they're not mutually exclusive to be safe
+    return {
+      isExclusive: false,
+      reason: 'Maximum recursion depth reached while checking schema exclusivity.',
+    };
+  }
   // Check top-level constraint exclusivity (types, formats, const values, enum values, and all constraints)
   const topLevelResult = arePropertiesMutuallyExclusive(sig1, sig2);
   if (!topLevelResult.isExclusive && topLevelResult.reason) {
@@ -333,14 +345,80 @@ function areSignaturesMutuallyExclusive(sig1: SchemaSignature, sig2: SchemaSigna
     const addlProps1 = sig1.additionalProperties;
     const addlProps2 = sig2.additionalProperties;
 
-    const allowsAdditional1 = addlProps1 !== false;
-    const allowsAdditional2 = addlProps2 !== false;
+    // Determine the type of additionalProperties for each schema
+    const isBoolean1 = typeof addlProps1 === 'boolean';
+    const isBoolean2 = typeof addlProps2 === 'boolean';
+    const isSchema1 = isPlainObject(addlProps1);
+    const isSchema2 = isPlainObject(addlProps2);
+
+    // Case 1: Both are booleans
+    if (isBoolean1 && isBoolean2) {
+      const allowsAdditional1 = addlProps1 !== false;
+      const allowsAdditional2 = addlProps2 !== false;
 
-    if (allowsAdditional1 !== allowsAdditional2) {
-      return {
-        isExclusive: false,
-        reason: 'One schema allows additional properties while the other does not.',
-      };
+      // If one allows and the other doesn't, they're not mutually exclusive (ambiguous)
+      if (allowsAdditional1 !== allowsAdditional2) {
+        return {
+          isExclusive: false,
+          reason: 'One schema allows additional properties while the other does not.',
+        };
+      }
+    }
+    // Case 2: One is boolean, the other is a schema/ref
+    else if ((isBoolean1 && isSchema2) || (isSchema1 && isBoolean2)) {
+      // If boolean is false, it means no additional properties allowed
+      // If the other has a schema, they conflict (one allows with constraints, one disallows)
+      const boolValue = isBoolean1 ? addlProps1 : addlProps2;
+
+      if (boolValue === false) {
+        return {
+          isExclusive: false,
+          reason:
+            'One schema disallows additional properties (false) while the other defines a schema for them.',
+        };
+      }
+      // If boolean is true, one allows any additional properties, the other allows with constraints
+      // This could be mutually exclusive if other constraints distinguish them, but the
+      // additionalProperties themselves don't provide discrimination
+    }
+    // Case 3: Both are schemas/refs
+    else if (isSchema1 && isSchema2) {
+      // Check if the additionalProperties schemas are mutually exclusive
+      const addlPropsResult = areSignaturesMutuallyExclusive(
+        addlProps1 as SchemaSignature,
+        addlProps2 as SchemaSignature,
+        depth + 1
+      );
+
+      // If the additionalProperties schemas are not mutually exclusive, it creates ambiguity
+      if (!addlPropsResult.isExclusive) {
+        return {
+          isExclusive: false,
+          reason: `Schemas have overlapping additionalProperties definitions. ${
+            addlPropsResult.reason || ''
+          }`,
+        };
+      }
+      // If they ARE mutually exclusive, this helps distinguish the schemas
+      // The additionalProperties provide discrimination, so schemas are mutually exclusive
+      return { isExclusive: true };
+    }
+    // Case 4: One or both are undefined (default is true in JSON Schema)
+    else {
+      // Default behavior: if undefined, it typically means additionalProperties: true
+      // If one is explicitly false and the other is undefined, they differ
+      if (addlProps1 === false || addlProps2 === false) {
+        const otherIsUndefined =
+          addlProps1 === false ? !isDefined(addlProps2) : !isDefined(addlProps1);
+
+        if (otherIsUndefined) {
+          return {
+            isExclusive: false,
+            reason:
+              'One schema explicitly disallows additional properties while the other allows them by default.',
+          };
+        }
+      }
     }
   }