fix: simplify isBinaryPayload to use encode="bytes" as binary signal (#3958)

* Initial plan

* update

(cherry picked from commit c2f06b1)

Co-authored-by: kazrael2119 <98569699+kazrael2119@users.noreply.github.com>

* style: format operationHelpers after isolated commit cherry-pick

Agent-Logs-Url: https://github.com/Azure/autorest.typescript/sessions/c5b9a83f-2fec-4bc7-b44b-2393f9d65a94

Co-authored-by: kazrael2119 <98569699+kazrael2119@users.noreply.github.com>

* docs: clarify isolated binary payload scenarios

Agent-Logs-Url: https://github.com/Azure/autorest.typescript/sessions/c5b9a83f-2fec-4bc7-b44b-2393f9d65a94

Co-authored-by: kazrael2119 <98569699+kazrael2119@users.noreply.github.com>

* Update operationUtil.ts

* Update operationUtil.ts

* Update bytesWithDifferentContentType.md

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: kazrael2119 <98569699+kazrael2119@users.noreply.github.com>
Co-authored-by: JialinHuang803 <139532647+JialinHuang803@users.noreply.github.com>

Author: 3 people

packages/typespec-ts/src/modular/helpers/operationHelpers.ts

@@ -452,7 +452,12 @@ export function getDeserializePrivateFunction(
             deserializedType,
             deserializedRoot,
             true,
-            isBinaryPayload(context, response.type!.__raw!, contentTypes)
+            isBinaryPayload(
+              context,
+              response.type!.__raw!,
+              contentTypes,
+              getEncodeForType(response.type!)
+            )
               ? "binary"
               : getEncodeForType(deserializedType)
           )}${multipartCastSuffix}`
@@ -1760,7 +1765,12 @@ function buildBodyParameter(
     bodyParameter.type,
     bodyNameExpression,
     !bodyParameter.optional,
-    isBinaryPayload(context, bodyParameter.__raw!, bodyParameter.contentTypes)
+    isBinaryPayload(
+      context,
+      bodyParameter.__raw!,
+      bodyParameter.contentTypes,
+      getEncodeForType(bodyParameter.type)
+    )
       ? "binary"
       : getEncodeForType(bodyParameter.type),
     undefined,
@@ -3266,7 +3276,10 @@ export function checkWrapNonModelReturn(
 
   // bytes with binary content type → binary wrap (isBinary=true)
   //   HLC: bytes → binary payload → separate binary handling
-  if (type.__raw && isBinaryPayload(context, type.__raw, contentTypes)) {
+  if (
+    type.__raw &&
+    isBinaryPayload(context, type.__raw, contentTypes, getEncodeForType(type))
+  ) {
     return { shouldWrap: true, isBinary: true };
   }
 

packages/typespec-ts/src/utils/operationUtil.ts

@@ -193,11 +193,18 @@ export function isDefinedStatusCode(statusCodes: HttpStatusCodesEntry) {
 export function isBinaryPayload(
   dpgContext: SdkContext,
   body: Type,
-  contentType: string | string[]
+  contentType: string | string[],
+  encode?: string
 ) {
   const knownMediaTypes: KnownMediaType[] = (
     Array.isArray(contentType) ? contentType : [contentType]
   ).map((ct) => knownMediaType(ct));
+
+  // When encode is "bytes" treat as binary.
+  if (encode === "bytes") {
+    return true;
+  }
+
   for (const type of knownMediaTypes) {
     if (type === KnownMediaType.Binary && isByteOrByteUnion(dpgContext, body)) {
       return true;