fix media upload resource, update aiplatform specs, remove tests for sttp3

Author: rolang

README.md

@@ -6,7 +6,7 @@
 
 Generates client code from Google's [disovery document](https://developers.google.com/discovery/v1/using) for your Scala (3) tech stack.  
 Currently it provides following configurations for generated code:
- - Http sources: [Sttp4](https://sttp.softwaremill.com/en/latest), [Sttp3](https://sttp.softwaremill.com/en/stable)
+ - Http sources: [Sttp4](https://sttp.softwaremill.com/en/latest)
  - JSON codecs: [Jsoniter](https://github.com/plokhotnyuk/jsoniter-scala), [ZioJson](https://zio.dev/zio-json)
  - JSON Array collection type: `List`, `Vector`, `Array`, `ZioChunk`
 
@@ -24,8 +24,8 @@ The generator can be used with any tool that can perform system calls to a comma
 See example under [example/generate.scala](./example/generate.scala).
 
 ```scala
-//> using scala 3.7.0
-//> using dep dev.rolang::gcp-codegen::0.0.5
+//> using scala 3.7.1
+//> using dep dev.rolang::gcp-codegen::0.0.7
 
 import gcp.codegen.*, java.nio.file.*, GeneratorConfig.*
 
@@ -59,7 +59,7 @@ See output in `example/out`.
 | -specs             | Can be `stdin` or a path to the JSON file. | | |
 | -out-dir           | Ouput directory | | |
 | -out-pkg           | Output package |  | |
-| -http-source       | Generated http source. | [Sttp4](https://sttp.softwaremill.com/en/latest), [Sttp3](https://sttp.softwaremill.com/en/stable) | |
+| -http-source       | Generated http source. | [Sttp4](https://sttp.softwaremill.com/en/stable) | |
 | -json-codec        | Generated JSON codec | [Jsoniter](https://github.com/plokhotnyuk/jsoniter-scala), [ZioJson](https://zio.dev/zio-json)  | |
 | -array-type        | Collection type for JSON arrays | `List`, `Vector`, `Array`, `ZioChunk` | `List` |
 | -include-resources | Optional resource filter. | | |

build.sbt

@@ -2,7 +2,7 @@ ThisBuild / description := "Google Cloud client code generator"
 ThisBuild / organization := "dev.rolang"
 ThisBuild / licenses := Seq(License.MIT)
 ThisBuild / homepage := Some(url("https://github.com/rolang/google-rest-api-codegen"))
-ThisBuild / scalaVersion := "3.3.6"
+ThisBuild / scalaVersion := "3.7.1"
 ThisBuild / version ~= { v => if (v.contains('+')) s"${v.replace('+', '-')}-SNAPSHOT" else v }
 ThisBuild / versionScheme := Some("early-semver")
 ThisBuild / scmInfo := Some(
@@ -37,9 +37,7 @@ lazy val noPublish = Seq(
   publish / skip := true
 )
 
-lazy val sttpClient4Version = "4.0.0-RC1"
-
-lazy val sttpClient3Version = "3.10.3"
+lazy val sttpClient4Version = "4.0.9"
 
 lazy val zioVersion = "2.1.16"
 
@@ -63,10 +61,7 @@ lazy val root = (project in file("."))
 // for supporting code inspection / testing of generated code via test_gen.sh script
 lazy val testLocal = (project in file("test-local"))
   .settings(
-    libraryDependencies ++= (
-      dependencyByConfig("Sttp4", "Jsoniter", "ZioChunk")
-        ++ dependencyByConfig("Sttp3", "ZioJson", "List")
-    )
+    libraryDependencies ++= dependencyByConfig("Sttp4", "Jsoniter", "ZioChunk")
   )
   .settings(noPublish)
 
@@ -97,7 +92,6 @@ lazy val cli = project
 
 def dependencyByConfig(httpSource: String, jsonCodec: String, arrayType: String): Seq[ModuleID] = {
   (httpSource match {
-    case "Sttp3" => Seq("com.softwaremill.sttp.client3" %% "core" % sttpClient3Version)
     case "Sttp4" => Seq("com.softwaremill.sttp.client4" %% "core" % sttpClient4Version)
     case other   => throw new InterruptedException(s"Invalid http source: $other")
   }) ++ (jsonCodec match {
@@ -125,7 +119,7 @@ lazy val testProjects: CompositeProject = new CompositeProject {
         "iamcredentials" -> "v1",
         "redis" -> "v1"
       )
-      httpSource <- Seq("Sttp4", "Sttp3")
+      httpSource <- Seq("Sttp4")
       jsonCodec <- Seq("ZioJson", "Jsoniter")
       arrayType <- Seq("ZioChunk", "List")
       id = s"test-$apiName-$apiVersion-${httpSource}-${jsonCodec}-${arrayType}".toLowerCase()
@@ -180,8 +174,6 @@ lazy val cliBinFile: File = {
 lazy val buildCliBinary = taskKey[File]("")
 buildCliBinary := {
   val built = (cli / Compile / nativeLinkReleaseFast).value
-  val destZip = new File(s"${cliBinFile.getPath()}.zip")
-
   IO.copyFile(built, cliBinFile)
   cliBinFile
 }
@@ -239,8 +231,17 @@ def codegenTask(
     val files = listFilesRec(List(outDir), Nil)
 
     // formatting (may need to find another way...)
-    logger.info(s"Formatting sources in $outPathRel...")
-    s"scala-cli fmt --scalafmt-conf=./.scalafmt.conf $outDir" ! ProcessLogger(_ => ()) // add logs when needed
+    val fmtCmd = s"scala-cli fmt --scalafmt-conf=./.scalafmt.conf ${outDir.absolutePath}"
+    logger.info(s"Formatting with '$fmtCmd'")
+    val fmtErrs = scala.collection.mutable.ListBuffer.empty[String]
+    fmtCmd ! ProcessLogger(
+      _ => (),
+      e => fmtErrs += e
+    ) match {
+      case 0 => ()
+      case c => throw new InterruptedException(s"Failure on code formatting: ${errs.mkString("\n")}")
+    }
+
     IO.delete(outDir / ".scala-build")
     logger.success(s"Formatting sources in $outPathRel done")
 

modules/cli/src/main/scala/cli.scala

@@ -1,8 +1,8 @@
 // for test runs using scala-cli
 //> using jvm system
-//> using scala 3.6.4
+//> using scala 3.7.1
 //> using file ../../../../core/shared/src/main/scala/codegen.scala
-//> using dep com.lihaoyi::upickle:4.1.0
+//> using dep com.lihaoyi::upickle:4.2.1
 
 package gcp.codegen.cli
 
@@ -13,24 +13,21 @@ import GeneratorConfig.*
 import scala.concurrent.Await
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.duration.*
-import scala.util.{Failure, Success}
 
 @main def run(args: String*) =
   argsToTask(args) match
     case Left(err) =>
       Console.err.println(s"Invalid arguments: $err")
       sys.exit(1)
     case Right(task) =>
-      Await
-        .ready(task.run, 30.seconds)
-        .onComplete {
-          case Failure(exception) =>
-            Console.err.println(s"Failure: ${exception.printStackTrace()}")
-            sys.exit(1)
-          case Success(files) =>
-            println(s"Generated ${files.length} files")
-            sys.exit(0)
-        }
+      try
+        val files = Await.result(task.run, 30.seconds)
+        println(s"Generated ${files.length} files")
+        sys.exit(0)
+      catch
+        case err: Throwable =>
+          Console.err.println(s"Failure: ${err.getMessage()}")
+          sys.exit(1)
 
 private def argsToTask(args: Seq[String]): Either[String, Task] =
   val argsMap = args.toList

modules/core/shared/src/main/scala/codegen.scala

@@ -235,11 +235,9 @@ def generateBySpec(
                   pkg = resourceKey.pkgName(resourcesPkg),
                   resourcesPkg = resourcesPkg,
                   schemasPkg = schemasPkg,
-                  baseUrl = specs.baseUrl,
                   resourceName = resourceName,
                   resource = resource,
                   httpSource = config.httpSource,
-                  jsonCodec = config.jsonCodec,
                   hasProps = p => specs.hasProps(p),
                   arrType = config.arrayType,
                   commonQueryParams = specs.queryParameters
@@ -312,11 +310,9 @@ def resourceCode(
     pkg: String,
     resourcesPkg: String,
     schemasPkg: String,
-    baseUrl: String,
     resourceName: String,
     resource: Resource,
     httpSource: HttpSource,
-    jsonCodec: JsonCodec,
     arrType: ArrayType,
     hasProps: SchemaPath => Boolean,
     commonQueryParams: Map[String, Parameter]
@@ -335,7 +331,7 @@ def resourceCode(
     "",
     s"object ${resourceName} {" +
       resource.methods
-        .map { (k, v) =>
+        .map { (k, method) =>
           def pathSegments(urlPath: String) =
             urlPath
               .split("/")
@@ -350,11 +346,11 @@ def resourceCode(
                     ) + "\")"
               )
 
-          val req = v.mediaUpload match
-            case None    => v.request.filter(_.schemaPath.forall(hasProps))
+          val req = method.mediaUploads match
+            case None    => method.request.filter(_.schemaPath.forall(hasProps))
             case Some(_) => None
 
-          val uploadProtocol = v.mediaUpload match
+          val uploadProtocol = method.mediaUploads match
             case Some(m) =>
               Some {
                 val protocols = m.protocols.keySet.toList.sortBy {
@@ -366,7 +362,7 @@ def resourceCode(
               }
             case None => None
 
-          val (requiredParams, optParams) = v.scalaParameters.partition(_._2.required)
+          val (requiredParams, optParams) = method.scalaParameters.partition(_._2.required)
           val params =
             requiredParams.map((n, t) => s"${toComment(t.description)}$n: ${t.scalaType(arrType)}") :::
               req.toList.map(r => s"request: ${r.scalaType(arrType)}") :::
@@ -375,17 +371,17 @@ def resourceCode(
               List(
                 s"endpointUrl: Uri = $rootPkg.baseUrl",
                 "commonQueryParams: QueryParameters = " + ((
-                  v.mediaUpload,
+                  method.mediaUploads,
                   commonQueryParams.collectFirst { case ("uploadType", Parameter(_, _, e: SchemaType.Enum, _, _)) => e }
                 ) match {
                   case (Some(m), Some(ut)) => s"""QueryParameters(uploadType = Some("${ut.values.head.value}"))"""
                   case _                   => "QueryParameters.empty"
                 })
               )
 
-          val setReqUri = v.mediaUpload match
+          val setReqUri = method.mediaUploads match
             case None =>
-              s"val requestUri = endpointUrl.addPathSegments(List(${pathSegments(v.urlPath).mkString(", ")}))"
+              s"val requestUri = endpointUrl.addPathSegments(List(${pathSegments(method.urlPath).mkString(", ")}))"
             case Some(m) =>
               List(
                 "val requestUri = uploadProtocol match {",
@@ -402,7 +398,7 @@ def resourceCode(
             case Some(_) => """.body(request.toJsonString)"""
 
           val queryParams = "\n    val params = " +
-            (v.scalaQueryParams match
+            (method.scalaQueryParams match
               case Nil => "commonQueryParams.value"
               case qParams =>
                 qParams
@@ -420,7 +416,7 @@ def resourceCode(
               case HttpSource.Sttp3 =>
                 s"RequestT[Identity, Either[ResponseException[String, Exception], $t], Any]"
 
-          val (resType, mapResponse) = v.response match
+          val (resType, mapResponse) = method.response match
             case Some(r) if r.schemaPath.forall(hasProps) =>
               val bodyType = r.scalaType(arrType)
 
@@ -432,7 +428,7 @@ def resourceCode(
 
           s"""|def ${toScalaName(k)}(\n${params.mkString(",\n")}): $resType = {$queryParams
               |  $setReqUri
-              |  resourceRequest.${v.httpMethod.toLowerCase()}(requestUri.addParams(params))$body$mapResponse
+              |  resourceRequest.${method.httpMethod.toLowerCase()}(requestUri.addParams(params))$body$mapResponse
               |}""".stripMargin
         }
         .mkString("\n", "\n\n", "\n") +
@@ -584,7 +580,7 @@ case class Method(
     parameterOrder: List[String],
     response: Option[SchemaType],
     request: Option[SchemaType] = None,
-    mediaUpload: Option[MediaUpload] = None
+    private val mediaUpload: Option[MediaUpload] = None
 ) {
   private lazy val flatPathParams: List[(String, Parameter)] = flatPath.toList.flatMap(p =>
     p.params.zipWithIndex.map((param, idx) =>
@@ -607,6 +603,20 @@ case class Method(
 
   def urlPath: String = flatPath.map(_.path).getOrElse(path)
 
+  def mediaUploads: Option[MediaUpload] = mediaUpload.map(m =>
+    m.copy(protocols =
+      m.protocols.view
+        .mapValues(p =>
+          // map the path to flatPath on placeholders with pattern like {+var_name} if the same is found in method path
+          // need a better solution for this
+          "\\{(\\+.*?)\\}".r.findAllIn(p.path).toList match
+            case v :: Nil if path.contains(v) => p.copy(path = flatPath.map(_.path).getOrElse(p.path))
+            case _                            => p
+        )
+        .toMap
+    )
+  )
+
   // filter out path params if flatPath params are given
   private lazy val pathParams: List[(String, Parameter)] =
     parameters.toList.filterNot((_, p) => flatPathParams.nonEmpty && p.location == "path")

test_gen.sh

@@ -4,7 +4,7 @@
 # runs code generator via scala-cli for given config
 # will ouput code under test-local/src/main/scala
 
-spec=pubsub
+spec=aiplatform
 json_codec=jsoniter
 http_source=sttp4
 array_type=list