Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 22 additions & 18 deletions .scalafmt.conf
Original file line number Diff line number Diff line change
@@ -1,28 +1,32 @@
version = 3.8.6
version = 3.11.1
runner.dialect = scala213

newlines.beforeMultilineDef = keep
newlines.topLevelStatements = [before]
newlines.beforeCurlyLambdaParams = multilineWithCaseOnly
newlines.afterCurlyLambdaParams = squash
newlines.implicitParamListModifierForce = [after]
newlines.avoidForSimpleOverflow = [tooLong]
newlines.avoidInResultType = true
newlines.sometimesBeforeColonInMethodReturnType = false
newlines.beforeTypeBounds = keep
align.openParenCallSite = true
align.openParenDefnSite = true

verticalMultiline.atDefnSite = true
verticalMultiline.arityThreshold = 10
maxColumn = 150
continuationIndent.defnSite = 2

spaces.inImportCurlyBraces = true
assumeStandardLibraryStripMargin = true
danglingParentheses.preset = true

rewrite.rules = [SortImports, RedundantParens, SortModifiers]

newlines.source = keep
newlines.afterCurlyLambda = preserve

spaces.inImportCurlyBraces = false

includeCurlyBraceInSelectChains = false
includeNoParensInSelectChains = false
optIn.breakChainOnFirstMethodDot = false

docstrings.style = Asterisk
docstrings.wrap = no
literals.long = Upper
literals.float = Upper
literals.double = Upper

literals.long=Upper
literals.float=Upper
literals.double=Upper
docstrings = JavaDoc
docstrings.style = keep
docstrings.wrap = no
docstrings.oneline = keep
docstrings.blankFirstLine = keep
15 changes: 8 additions & 7 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -204,9 +204,9 @@ lazy val docs = project
.dependsOn(dataset, cats, ml)

def sparkDependencies(
sparkVersion: String,
scope: Configuration = Provided
) = Seq(
sparkVersion: String,
scope: Configuration = Provided
) = Seq(
libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-core" % sparkVersion % scope,
"org.apache.spark" %% "spark-sql" % sparkVersion % scope
Expand Down Expand Up @@ -378,10 +378,11 @@ lazy val spark40Settings = Seq[Setting[_]](
lazy val spark34Settings = Seq[Setting[_]](
tlVersionIntroduced := Map("2.12" -> "0.14.1", "2.13" -> "0.14.1"),
mimaPreviousArtifacts := Set(
organization.value %% moduleName.value
.split("-")
.dropRight(1)
.mkString("-") % "0.14.1"
organization.value %%
moduleName.value
.split("-")
.dropRight(1)
.mkString("-") % "0.14.1"
)
)

Expand Down
4 changes: 2 additions & 2 deletions cats/src/main/scala/frameless/cats/FramelessSyntax.scala
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ trait FramelessSyntax extends frameless.FramelessSyntax {
def withLocalProperty(key: String, value: String): F[A] =
for {
session <- ask
_ <- delay(session.sparkContext.setLocalProperty(key, value))
a <- fa
_ <- delay(session.sparkContext.setLocalProperty(key, value))
a <- fa
} yield a

def withGroupId(groupId: String): F[A] = withLocalProperty("spark.jobGroup.id", groupId)
Expand Down
6 changes: 3 additions & 3 deletions cats/src/main/scala/frameless/cats/implicits.scala
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,9 @@ object outer {
def combine(lhs: RDD[(K, V)], rhs: RDD[(K, V)]): RDD[(K, V)] =
lhs.fullOuterJoin(rhs).mapValues {
case (Some(x), Some(y)) => x |+| y
case (None, Some(y)) => y
case (Some(x), None) => x
case (None, None) => m.empty
case (None, Some(y)) => y
case (Some(x), None) => x
case (None, None) => m.empty
}
}
}
12 changes: 6 additions & 6 deletions cats/src/test/scala/frameless/cats/test.scala
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import org.scalatest.matchers.should.Matchers
import org.scalatest.propspec.AnyPropSpec

trait SparkTests {
val appID: String = new java.util.Date().toString + math.floor(math.random() * 10E4).toLong.toString
val appID: String = new java.util.Date().toString + math.floor(math.random() * 10e4).toLong.toString

val conf: SparkConf = new SparkConf()
.setMaster("local[*]")
Expand Down Expand Up @@ -68,7 +68,7 @@ class Test extends AnyPropSpec with Matchers with ScalaCheckPropertyChecks with
PropertyCheckConfiguration(minSize = PosInt(10))

property("spark is working") {
sc.parallelize(Seq(1, 2, 3)).collect() shouldBe Array(1,2,3)
sc.parallelize(Seq(1, 2, 3)).collect() shouldBe Array(1, 2, 3)
}

property("inner pairwise monoid") {
Expand Down Expand Up @@ -120,10 +120,10 @@ class Test extends AnyPropSpec with Matchers with ScalaCheckPropertyChecks with

property("pair rdd numeric commutative semigroup example") {
import frameless.cats.implicits._
val seq = Seq( ("a",2), ("b",3), ("d",6), ("b",2), ("d",1) )
val seq = Seq(("a", 2), ("b", 3), ("d", 6), ("b", 2), ("d", 1))
val rdd = seq.toRdd
rdd.cminByKey.collect().toSeq should contain theSameElementsAs Seq( ("a",2), ("b",2), ("d",1) )
rdd.cmaxByKey.collect().toSeq should contain theSameElementsAs Seq( ("a",2), ("b",3), ("d",6) )
rdd.csumByKey.collect().toSeq should contain theSameElementsAs Seq( ("a",2), ("b",5), ("d",7) )
rdd.cminByKey.collect().toSeq should contain theSameElementsAs Seq(("a", 2), ("b", 2), ("d", 1))
rdd.cmaxByKey.collect().toSeq should contain theSameElementsAs Seq(("a", 2), ("b", 3), ("d", 6))
rdd.csumByKey.collect().toSeq should contain theSameElementsAs Seq(("a", 2), ("b", 5), ("d", 7))
}
}
8 changes: 4 additions & 4 deletions core/src/main/scala/frameless/CatalystAverageable.scala
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ object CatalystAverageable {
private[this] def of[In, Out]: CatalystAverageable[In, Out] = theInstance.asInstanceOf[CatalystAverageable[In, Out]]

implicit val framelessAverageableBigDecimal: CatalystAverageable[BigDecimal, BigDecimal] = of[BigDecimal, BigDecimal]
implicit val framelessAverageableDouble: CatalystAverageable[Double, Double] = of[Double, Double]
implicit val framelessAverageableLong: CatalystAverageable[Long, Double] = of[Long, Double]
implicit val framelessAverageableInt: CatalystAverageable[Int, Double] = of[Int, Double]
implicit val framelessAverageableShort: CatalystAverageable[Short, Double] = of[Short, Double]
implicit val framelessAverageableDouble: CatalystAverageable[Double, Double] = of[Double, Double]
implicit val framelessAverageableLong: CatalystAverageable[Long, Double] = of[Long, Double]
implicit val framelessAverageableInt: CatalystAverageable[Int, Double] = of[Int, Double]
implicit val framelessAverageableShort: CatalystAverageable[Short, Double] = of[Short, Double]
}
10 changes: 5 additions & 5 deletions core/src/main/scala/frameless/CatalystBitShift.scala
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@ object CatalystBitShift {
private[this] val theInstance = new CatalystBitShift[Any, Any] {}
private[this] def of[In, Out]: CatalystBitShift[In, Out] = theInstance.asInstanceOf[CatalystBitShift[In, Out]]

implicit val framelessBitShiftBigDecimal: CatalystBitShift[BigDecimal, Int] = of[BigDecimal, Int]
implicit val framelessBitShiftDouble : CatalystBitShift[Byte, Int] = of[Byte, Int]
implicit val framelessBitShiftInt : CatalystBitShift[Short, Int] = of[Short, Int]
implicit val framelessBitShiftLong : CatalystBitShift[Int, Int] = of[Int, Int]
implicit val framelessBitShiftShort : CatalystBitShift[Long, Long] = of[Long, Long]
implicit val framelessBitShiftBigDecimal: CatalystBitShift[BigDecimal, Int] = of[BigDecimal, Int]
implicit val framelessBitShiftDouble: CatalystBitShift[Byte, Int] = of[Byte, Int]
implicit val framelessBitShiftInt: CatalystBitShift[Short, Int] = of[Short, Int]
implicit val framelessBitShiftLong: CatalystBitShift[Int, Int] = of[Int, Int]
implicit val framelessBitShiftShort: CatalystBitShift[Long, Long] = of[Long, Long]
}
27 changes: 13 additions & 14 deletions core/src/main/scala/frameless/CatalystCast.scala
Original file line number Diff line number Diff line change
Expand Up @@ -8,25 +8,25 @@ object CatalystCast {

implicit def framelessCastToString[T]: CatalystCast[T, String] = of[T, String]

implicit def framelessNumericToLong [A: CatalystNumeric]: CatalystCast[A, Long] = of[A, Long]
implicit def framelessNumericToInt [A: CatalystNumeric]: CatalystCast[A, Int] = of[A, Int]
implicit def framelessNumericToShort [A: CatalystNumeric]: CatalystCast[A, Short] = of[A, Short]
implicit def framelessNumericToByte [A: CatalystNumeric]: CatalystCast[A, Byte] = of[A, Byte]
implicit def framelessNumericToLong[A: CatalystNumeric]: CatalystCast[A, Long] = of[A, Long]
implicit def framelessNumericToInt[A: CatalystNumeric]: CatalystCast[A, Int] = of[A, Int]
implicit def framelessNumericToShort[A: CatalystNumeric]: CatalystCast[A, Short] = of[A, Short]
implicit def framelessNumericToByte[A: CatalystNumeric]: CatalystCast[A, Byte] = of[A, Byte]
implicit def framelessNumericToDecimal[A: CatalystNumeric]: CatalystCast[A, BigDecimal] = of[A, BigDecimal]
implicit def framelessNumericToDouble [A: CatalystNumeric]: CatalystCast[A, Double] = of[A, Double]
implicit def framelessNumericToDouble[A: CatalystNumeric]: CatalystCast[A, Double] = of[A, Double]

implicit def framelessBooleanToNumeric[A: CatalystNumeric]: CatalystCast[Boolean, A] = of[Boolean, A]

// doesn't make any sense to include:
// - sqlDateToBoolean: always None
// - sqlTimestampToBoolean: compares us to 0
implicit val framelessStringToBoolean : CatalystCast[String, Option[Boolean]] = of[String, Option[Boolean]]
implicit val framelessLongToBoolean : CatalystCast[Long, Boolean] = of[Long, Boolean]
implicit val framelessIntToBoolean : CatalystCast[Int, Boolean] = of[Int, Boolean]
implicit val framelessShortToBoolean : CatalystCast[Short, Boolean] = of[Short, Boolean]
implicit val framelessByteToBoolean : CatalystCast[Byte, Boolean] = of[Byte, Boolean]
implicit val framelessBigDecimalToBoolean: CatalystCast[BigDecimal, Boolean] = of[BigDecimal, Boolean]
implicit val framelessDoubleToBoolean : CatalystCast[Double, Boolean] = of[Double, Boolean]
implicit val framelessStringToBoolean: CatalystCast[String, Option[Boolean]] = of[String, Option[Boolean]]
implicit val framelessLongToBoolean: CatalystCast[Long, Boolean] = of[Long, Boolean]
implicit val framelessIntToBoolean: CatalystCast[Int, Boolean] = of[Int, Boolean]
implicit val framelessShortToBoolean: CatalystCast[Short, Boolean] = of[Short, Boolean]
implicit val framelessByteToBoolean: CatalystCast[Byte, Boolean] = of[Byte, Boolean]
implicit val framelessBigDecimalToBoolean: CatalystCast[BigDecimal, Boolean] = of[BigDecimal, Boolean]
implicit val framelessDoubleToBoolean: CatalystCast[Double, Boolean] = of[Double, Boolean]

// TODO

Expand All @@ -38,9 +38,8 @@ object CatalystCast {
// implicit object stringToLong extends CatalystCast[String, Option[Long]]
// implicit object stringToSqlDate extends CatalystCast[String, Option[SQLDate]]


// needs verification:
//implicit object sqlTimestampToSqlDate extends CatalystCast[SQLTimestamp, SQLDate]
// implicit object sqlTimestampToSqlDate extends CatalystCast[SQLTimestamp, SQLDate]

// needs verification:
// implicit object sqlTimestampToDecimal extends CatalystCast[SQLTimestamp, BigDecimal]
Expand Down
6 changes: 3 additions & 3 deletions core/src/main/scala/frameless/CatalystCollection.scala
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ object CatalystCollection {
private[this] val theInstance = new CatalystCollection[Any] {}
private[this] def of[A[_]]: CatalystCollection[A] = theInstance.asInstanceOf[CatalystCollection[A]]

implicit val arrayObject : CatalystCollection[Array] = of[Array]
implicit val seqObject : CatalystCollection[Seq] = of[Seq]
implicit val listObject : CatalystCollection[List] = of[List]
implicit val arrayObject: CatalystCollection[Array] = of[Array]
implicit val seqObject: CatalystCollection[Seq] = of[Seq]
implicit val listObject: CatalystCollection[List] = of[List]
implicit val vectorObject: CatalystCollection[Vector] = of[Vector]
}
10 changes: 5 additions & 5 deletions core/src/main/scala/frameless/CatalystDivisible.scala
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@ object CatalystDivisible {
private[this] def of[In, Out]: CatalystDivisible[In, Out] = theInstance.asInstanceOf[CatalystDivisible[In, Out]]

implicit val framelessDivisibleBigDecimal: CatalystDivisible[BigDecimal, BigDecimal] = of[BigDecimal, BigDecimal]
implicit val framelessDivisibleDouble : CatalystDivisible[Double, Double] = of[Double, Double]
implicit val framelessDivisibleInt : CatalystDivisible[Int, Double] = of[Int, Double]
implicit val framelessDivisibleLong : CatalystDivisible[Long, Double] = of[Long, Double]
implicit val framelessDivisibleByte : CatalystDivisible[Byte, Double] = of[Byte, Double]
implicit val framelessDivisibleShort : CatalystDivisible[Short, Double] = of[Short, Double]
implicit val framelessDivisibleDouble: CatalystDivisible[Double, Double] = of[Double, Double]
implicit val framelessDivisibleInt: CatalystDivisible[Int, Double] = of[Int, Double]
implicit val framelessDivisibleLong: CatalystDivisible[Long, Double] = of[Long, Double]
implicit val framelessDivisibleByte: CatalystDivisible[Byte, Double] = of[Byte, Double]
implicit val framelessDivisibleShort: CatalystDivisible[Short, Double] = of[Short, Double]
}
14 changes: 7 additions & 7 deletions core/src/main/scala/frameless/CatalystIsin.scala
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,11 @@ trait CatalystIsin[A]

object CatalystIsin {
implicit object framelessBigDecimal extends CatalystIsin[BigDecimal]
implicit object framelessByte extends CatalystIsin[Byte]
implicit object framelessDouble extends CatalystIsin[Double]
implicit object framelessFloat extends CatalystIsin[Float]
implicit object framelessInt extends CatalystIsin[Int]
implicit object framelessLong extends CatalystIsin[Long]
implicit object framelessShort extends CatalystIsin[Short]
implicit object framelesssString extends CatalystIsin[String]
implicit object framelessByte extends CatalystIsin[Byte]
implicit object framelessDouble extends CatalystIsin[Double]
implicit object framelessFloat extends CatalystIsin[Float]
implicit object framelessInt extends CatalystIsin[Int]
implicit object framelessLong extends CatalystIsin[Long]
implicit object framelessShort extends CatalystIsin[Short]
implicit object framelesssString extends CatalystIsin[String]
}
5 changes: 2 additions & 3 deletions core/src/main/scala/frameless/CatalystNaN.scala
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ object CatalystNaN {
private[this] val theInstance = new CatalystNaN[Any] {}
private[this] def of[A]: CatalystNaN[A] = theInstance.asInstanceOf[CatalystNaN[A]]

implicit val framelessFloatNaN : CatalystNaN[Float] = of[Float]
implicit val framelessDoubleNaN : CatalystNaN[Double] = of[Double]
implicit val framelessFloatNaN: CatalystNaN[Float] = of[Float]
implicit val framelessDoubleNaN: CatalystNaN[Double] = of[Double]
}

10 changes: 5 additions & 5 deletions core/src/main/scala/frameless/CatalystNumeric.scala
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@ object CatalystNumeric {
private[this] def of[A]: CatalystNumeric[A] = theInstance.asInstanceOf[CatalystNumeric[A]]

implicit val framelessbigDecimalNumeric: CatalystNumeric[BigDecimal] = of[BigDecimal]
implicit val framelessbyteNumeric : CatalystNumeric[Byte] = of[Byte]
implicit val framelessdoubleNumeric : CatalystNumeric[Double] = of[Double]
implicit val framelessintNumeric : CatalystNumeric[Int] = of[Int]
implicit val framelesslongNumeric : CatalystNumeric[Long] = of[Long]
implicit val framelessshortNumeric : CatalystNumeric[Short] = of[Short]
implicit val framelessbyteNumeric: CatalystNumeric[Byte] = of[Byte]
implicit val framelessdoubleNumeric: CatalystNumeric[Double] = of[Double]
implicit val framelessintNumeric: CatalystNumeric[Int] = of[Int]
implicit val framelesslongNumeric: CatalystNumeric[Long] = of[Long]
implicit val framelessshortNumeric: CatalystNumeric[Short] = of[Short]
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,11 @@ object CatalystNumericWithJavaBigDecimal {
private[this] val theInstance = new CatalystNumericWithJavaBigDecimal[Any, Any] {}
private[this] def of[In, Out]: CatalystNumericWithJavaBigDecimal[In, Out] = theInstance.asInstanceOf[CatalystNumericWithJavaBigDecimal[In, Out]]

implicit val framelessAbsoluteBigDecimal: CatalystNumericWithJavaBigDecimal[BigDecimal, java.math.BigDecimal] = of[BigDecimal, java.math.BigDecimal]
implicit val framelessAbsoluteDouble : CatalystNumericWithJavaBigDecimal[Double, Double] = of[Double, Double]
implicit val framelessAbsoluteInt : CatalystNumericWithJavaBigDecimal[Int, Int] = of[Int, Int]
implicit val framelessAbsoluteLong : CatalystNumericWithJavaBigDecimal[Long, Long] = of[Long, Long]
implicit val framelessAbsoluteShort : CatalystNumericWithJavaBigDecimal[Short, Short] = of[Short, Short]
implicit val framelessAbsoluteByte : CatalystNumericWithJavaBigDecimal[Byte, Byte] = of[Byte, Byte]
implicit val framelessAbsoluteBigDecimal: CatalystNumericWithJavaBigDecimal[BigDecimal, java.math.BigDecimal] = of[BigDecimal, java.math.BigDecimal]
implicit val framelessAbsoluteDouble: CatalystNumericWithJavaBigDecimal[Double, Double] = of[Double, Double]
implicit val framelessAbsoluteInt: CatalystNumericWithJavaBigDecimal[Int, Int] = of[Int, Int]
implicit val framelessAbsoluteLong: CatalystNumericWithJavaBigDecimal[Long, Long] = of[Long, Long]
implicit val framelessAbsoluteShort: CatalystNumericWithJavaBigDecimal[Short, Short] = of[Short, Short]
implicit val framelessAbsoluteByte: CatalystNumericWithJavaBigDecimal[Byte, Byte] = of[Byte, Byte]

}
}
44 changes: 21 additions & 23 deletions core/src/main/scala/frameless/CatalystOrdered.scala
Original file line number Diff line number Diff line change
Expand Up @@ -13,30 +13,28 @@ object CatalystOrdered {
private[this] val theInstance = new CatalystOrdered[Any] {}
private[this] def of[A]: CatalystOrdered[A] = theInstance.asInstanceOf[CatalystOrdered[A]]

implicit val framelessIntOrdered : CatalystOrdered[Int] = of[Int]
implicit val framelessBooleanOrdered : CatalystOrdered[Boolean] = of[Boolean]
implicit val framelessByteOrdered : CatalystOrdered[Byte] = of[Byte]
implicit val framelessShortOrdered : CatalystOrdered[Short] = of[Short]
implicit val framelessLongOrdered : CatalystOrdered[Long] = of[Long]
implicit val framelessFloatOrdered : CatalystOrdered[Float] = of[Float]
implicit val framelessDoubleOrdered : CatalystOrdered[Double] = of[Double]
implicit val framelessBigDecimalOrdered : CatalystOrdered[BigDecimal] = of[BigDecimal]
implicit val framelessSQLDateOrdered : CatalystOrdered[SQLDate] = of[SQLDate]
implicit val framelessIntOrdered: CatalystOrdered[Int] = of[Int]
implicit val framelessBooleanOrdered: CatalystOrdered[Boolean] = of[Boolean]
implicit val framelessByteOrdered: CatalystOrdered[Byte] = of[Byte]
implicit val framelessShortOrdered: CatalystOrdered[Short] = of[Short]
implicit val framelessLongOrdered: CatalystOrdered[Long] = of[Long]
implicit val framelessFloatOrdered: CatalystOrdered[Float] = of[Float]
implicit val framelessDoubleOrdered: CatalystOrdered[Double] = of[Double]
implicit val framelessBigDecimalOrdered: CatalystOrdered[BigDecimal] = of[BigDecimal]
implicit val framelessSQLDateOrdered: CatalystOrdered[SQLDate] = of[SQLDate]
implicit val framelessSQLTimestampOrdered: CatalystOrdered[SQLTimestamp] = of[SQLTimestamp]
implicit val framelessStringOrdered : CatalystOrdered[String] = of[String]
implicit val framelessInstantOrdered : CatalystOrdered[Instant] = of[Instant]
implicit val framelessDurationOrdered : CatalystOrdered[Duration] = of[Duration]
implicit val framelessPeriodOrdered : CatalystOrdered[Period] = of[Period]
implicit val framelessStringOrdered: CatalystOrdered[String] = of[String]
implicit val framelessInstantOrdered: CatalystOrdered[Instant] = of[Instant]
implicit val framelessDurationOrdered: CatalystOrdered[Duration] = of[Duration]
implicit val framelessPeriodOrdered: CatalystOrdered[Period] = of[Period]

implicit def injectionOrdered[A, B]
(implicit
i0: Injection[A, B],
i1: CatalystOrdered[B]
): CatalystOrdered[A] = of[A]
implicit def injectionOrdered[A, B](implicit
i0: Injection[A, B],
i1: CatalystOrdered[B]
): CatalystOrdered[A] = of[A]

implicit def deriveGeneric[G, H <: HList]
(implicit
i0: Generic.Aux[G, H],
i1: Lazy[LiftAll[CatalystOrdered, H]]
): CatalystOrdered[G] = of[G]
implicit def deriveGeneric[G, H <: HList](implicit
i0: Generic.Aux[G, H],
i1: Lazy[LiftAll[CatalystOrdered, H]]
): CatalystOrdered[G] = of[G]
}
Loading
Loading