Skip to content

Commit 3d5b20e

Browse files
pflookypflooky
committed
fix: revert back plan models changes and unified yaml, cleanup imports
1 parent 382a8fb commit 3d5b20e

75 files changed

Lines changed: 564 additions & 7137 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎api/src/main/scala/io/github/datacatering/datacaterer/api/DataCatererConfigurationBuilder.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ package io.github.datacatering.datacaterer.api
33
import com.softwaremill.quicklens.ModifyPimp
44
import io.github.datacatering.datacaterer.api.connection.{BigQueryBuilder, CassandraBuilder, ConnectionTaskBuilder, FileBuilder, HttpBuilder, KafkaBuilder, MySqlBuilder, NoopBuilder, PostgresBuilder, RabbitmqBuilder, SolaceBuilder}
55
import io.github.datacatering.datacaterer.api.converter.Converters.toScalaMap
6-
import io.github.datacatering.datacaterer.api.model.Constants.{BIGQUERY_WRITE_METHOD, _}
6+
import io.github.datacatering.datacaterer.api.model.Constants._
77
import io.github.datacatering.datacaterer.api.model.DataCatererConfiguration
88

99
import scala.annotation.varargs

‎api/src/main/scala/io/github/datacatering/datacaterer/api/ForeignKeyConfigBuilders.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
package io.github.datacatering.datacaterer.api
22

33
import com.softwaremill.quicklens.ModifyPimp
4-
import io.github.datacatering.datacaterer.api.model.{CardinalityConfig, NullabilityConfig}
54
import io.github.datacatering.datacaterer.api.model.Constants._
5+
import io.github.datacatering.datacaterer.api.model.{CardinalityConfig, NullabilityConfig}
66

77
/**
88
* Builder for CardinalityConfig to control foreign key cardinality in relationships.

‎api/src/main/scala/io/github/datacatering/datacaterer/api/MetadataSourceBuilder.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package io.github.datacatering.datacaterer.api
22

33
import com.softwaremill.quicklens.ModifyPimp
44
import io.github.datacatering.datacaterer.api.converter.Converters.toScalaMap
5-
import io.github.datacatering.datacaterer.api.model.Constants.{CONFLUENT_SCHEMA_REGISTRY_ID, CONFLUENT_SCHEMA_REGISTRY_SUBJECT, CONFLUENT_SCHEMA_REGISTRY_VERSION, DATA_CONTRACT_FILE, DATA_CONTRACT_SCHEMA, GREAT_EXPECTATIONS_FILE, JSON_SCHEMA_FILE, METADATA_SOURCE_URL, OPEN_LINEAGE_DATASET, OPEN_LINEAGE_NAMESPACE, OPEN_METADATA_API_VERSION, OPEN_METADATA_AUTH_TYPE, OPEN_METADATA_AUTH_TYPE_OPEN_METADATA, OPEN_METADATA_DEFAULT_API_VERSION, OPEN_METADATA_HOST, OPEN_METADATA_JWT_TOKEN, SCHEMA_LOCATION, YAML_PLAN_FILE, YAML_STEP_NAME, YAML_TASK_FILE, YAML_TASK_NAME}
5+
import io.github.datacatering.datacaterer.api.model.Constants._
66
import io.github.datacatering.datacaterer.api.model.{ConfluentSchemaRegistrySource, DataContractCliSource, GreatExpectationsSource, JsonSchemaSource, MarquezMetadataSource, MetadataSource, OpenAPISource, OpenDataContractStandardSource, OpenMetadataSource, YamlPlanSource, YamlTaskSource}
77

88
case class MetadataSourceBuilder(metadataSource: MetadataSource = MarquezMetadataSource()) {

‎api/src/main/scala/io/github/datacatering/datacaterer/api/PlanBuilder.scala‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -97,22 +97,6 @@ case class PlanBuilder(plan: Plan = Plan(), tasks: List[TasksBuilder] = List())
9797
relations: java.util.List[(ConnectionTaskBuilder[_], java.util.List[String])]): PlanBuilder =
9898
addForeignKeyRelationship(foreignKey, toScalaList(relations).map(r => toForeignKeyRelation(r._1, toScalaList(r._2))): _*)
9999

100-
def addForeignKeyRelationship(foreignKey: ForeignKeyRelation,
101-
generationLinks: List[ForeignKeyRelation],
102-
cardinality: CardinalityConfigBuilder): PlanBuilder =
103-
this.modify(_.plan.sinkOptions).setTo(Some(getSinkOpt.foreignKey(foreignKey, generationLinks, cardinality).sinkOptions))
104-
105-
def addForeignKeyRelationship(foreignKey: ForeignKeyRelation,
106-
generationLinks: List[ForeignKeyRelation],
107-
nullability: NullabilityConfigBuilder): PlanBuilder =
108-
this.modify(_.plan.sinkOptions).setTo(Some(getSinkOpt.foreignKey(foreignKey, generationLinks, nullability).sinkOptions))
109-
110-
def addForeignKeyRelationship(foreignKey: ForeignKeyRelation,
111-
generationLinks: List[ForeignKeyRelation],
112-
cardinality: CardinalityConfigBuilder,
113-
nullability: NullabilityConfigBuilder): PlanBuilder =
114-
this.modify(_.plan.sinkOptions).setTo(Some(getSinkOpt.foreignKey(foreignKey, generationLinks, cardinality, nullability).sinkOptions))
115-
116100
private def toForeignKeyRelation(connectionTaskBuilder: ConnectionTaskBuilder[_], fields: List[String], isDeleteFk: Boolean = false): ForeignKeyRelation = {
117101
val dataSource = connectionTaskBuilder.connectionConfigWithTaskBuilder.dataSourceName
118102
val fieldNames = fields.mkString(",")

‎api/src/main/scala/io/github/datacatering/datacaterer/api/SinkOptionsBuilder.scala‎

Lines changed: 1 addition & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package io.github.datacatering.datacaterer.api
22

33
import com.softwaremill.quicklens.ModifyPimp
4-
import io.github.datacatering.datacaterer.api.model.{CardinalityConfig, ForeignKey, ForeignKeyRelation, ManyToManyRelation, NullabilityConfig, SinkOptions}
4+
import io.github.datacatering.datacaterer.api.model.{ForeignKey, ForeignKeyRelation, SinkOptions}
55

66
import scala.annotation.varargs
77

@@ -68,108 +68,4 @@ case class SinkOptionsBuilder(sinkOptions: SinkOptions = SinkOptions()) {
6868
def foreignKey(foreignKey: ForeignKeyRelation, generationLinks: List[ForeignKeyRelation]): SinkOptionsBuilder =
6969
this.foreignKey(foreignKey, generationLinks: _*)
7070

71-
/**
72-
* Define a foreign key relationship with enhanced configuration options.
73-
*
74-
* @param foreignKey Base foreign key
75-
* @param generationLinks Foreign key relations for data generation
76-
* @param deleteLinks Foreign key relations for data deletion
77-
* @param relationshipType Optional relationship type: "one-to-one", "one-to-many", "many-to-many"
78-
* @param cardinality Optional cardinality configuration
79-
* @param nullability Optional nullability configuration
80-
* @param generationMode Optional generation mode: "all-exist", "all-combinations", "partial"
81-
* @return SinkOptionsBuilder
82-
*/
83-
def foreignKey(foreignKey: ForeignKeyRelation,
84-
generationLinks: List[ForeignKeyRelation],
85-
deleteLinks: List[ForeignKeyRelation],
86-
relationshipType: Option[String]): SinkOptionsBuilder =
87-
this.modify(_.sinkOptions.foreignKeys)(_ ++ List(ForeignKey(
88-
foreignKey, generationLinks, deleteLinks,
89-
relationshipType
90-
)))
91-
92-
/**
93-
* DEPRECATED: Cardinality is now configured per-target on ForeignKeyRelation.
94-
* Use target.copy(cardinality = Some(CardinalityConfig(...))) instead.
95-
*
96-
* @param foreignKey Base foreign key
97-
* @param generationLinks Foreign key relations for data generation
98-
* @param cardinality Cardinality configuration builder
99-
* @return SinkOptionsBuilder
100-
*/
101-
@deprecated("Configure cardinality on individual ForeignKeyRelation targets instead", "0.18.0")
102-
def foreignKey(foreignKey: ForeignKeyRelation,
103-
generationLinks: List[ForeignKeyRelation],
104-
cardinality: CardinalityConfigBuilder): SinkOptionsBuilder = {
105-
// Apply cardinality to all targets
106-
val targetsWithCard = generationLinks.map(_.copy(cardinality = Some(cardinality.config)))
107-
this.foreignKey(foreignKey, targetsWithCard, List())
108-
}
109-
110-
/**
111-
* DEPRECATED: Nullability is now configured per-target on ForeignKeyRelation.
112-
* Use target.copy(nullability = Some(NullabilityConfig(...))) instead.
113-
*
114-
* @param foreignKey Base foreign key
115-
* @param generationLinks Foreign key relations for data generation
116-
* @param nullability Nullability configuration builder
117-
* @return SinkOptionsBuilder
118-
*/
119-
@deprecated("Configure nullability on individual ForeignKeyRelation targets instead", "0.18.0")
120-
def foreignKey(foreignKey: ForeignKeyRelation,
121-
generationLinks: List[ForeignKeyRelation],
122-
nullability: NullabilityConfigBuilder): SinkOptionsBuilder = {
123-
// Apply nullability to all targets
124-
val targetsWithNull = generationLinks.map(_.copy(nullability = Some(nullability.config)))
125-
this.foreignKey(foreignKey, targetsWithNull, List())
126-
}
127-
128-
/**
129-
* DEPRECATED: Cardinality and nullability are now configured per-target on ForeignKeyRelation.
130-
* Use target.copy(cardinality = Some(...), nullability = Some(...)) instead.
131-
*
132-
* @param foreignKey Base foreign key
133-
* @param generationLinks Foreign key relations for data generation
134-
* @param cardinality Cardinality configuration builder
135-
* @param nullability Nullability configuration builder
136-
* @return SinkOptionsBuilder
137-
*/
138-
@deprecated("Configure cardinality and nullability on individual ForeignKeyRelation targets instead", "0.18.0")
139-
def foreignKey(foreignKey: ForeignKeyRelation,
140-
generationLinks: List[ForeignKeyRelation],
141-
cardinality: CardinalityConfigBuilder,
142-
nullability: NullabilityConfigBuilder): SinkOptionsBuilder = {
143-
// Apply both to all targets
144-
val targetsWithConfig = generationLinks.map(_.copy(
145-
cardinality = Some(cardinality.config),
146-
nullability = Some(nullability.config)
147-
))
148-
this.foreignKey(foreignKey, targetsWithConfig, List())
149-
}
150-
151-
/**
152-
* Define a many-to-many relationship using junction table pattern.
153-
*
154-
* @param leftSource Left side of the relationship (e.g., students)
155-
* @param rightSource Right side of the relationship (e.g., courses)
156-
* @param junctionTable Junction/bridge table (e.g., enrollments)
157-
* @param leftCardinality Optional cardinality config for left side
158-
* @param rightCardinality Optional cardinality config for right side
159-
* @return SinkOptionsBuilder
160-
*/
161-
def manyToManyRelationship(leftSource: ForeignKeyRelation,
162-
rightSource: ForeignKeyRelation,
163-
junctionTable: ForeignKeyRelation,
164-
leftCardinality: Option[CardinalityConfig] = None,
165-
rightCardinality: Option[CardinalityConfig] = None): SinkOptionsBuilder = {
166-
// Create two foreign key relationships: left->junction and right->junction
167-
// Apply cardinality to junction table targets
168-
val leftJunction = leftCardinality.map(c => junctionTable.copy(cardinality = Some(c))).getOrElse(junctionTable)
169-
val rightJunction = rightCardinality.map(c => junctionTable.copy(cardinality = Some(c))).getOrElse(junctionTable)
170-
171-
this
172-
.foreignKey(leftSource, List(leftJunction), List())
173-
.foreignKey(rightSource, List(rightJunction), List())
174-
}
17571
}

‎api/src/main/scala/io/github/datacatering/datacaterer/api/YamlBuilder.scala‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
package io.github.datacatering.datacaterer.api
22

33
import com.softwaremill.quicklens.ModifyPimp
4-
import io.github.datacatering.datacaterer.api.converter.Converters.toScalaMap
54
import io.github.datacatering.datacaterer.api.model.Constants.{YAML_PLAN_FILE, YAML_STEP_NAME, YAML_TASK_FILE, YAML_TASK_NAME}
6-
import io.github.datacatering.datacaterer.api.model.{Plan, Task}
75

86
/**
97
* Builds configurations by loading existing YAML plan or task files and allowing override of specific configurations.

‎api/src/main/scala/io/github/datacatering/datacaterer/api/model/ConfigModels.scala‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ case class FlagsConfig(
1717
enableAlerts: Boolean = DEFAULT_ENABLE_ALERTS,
1818
enableUniqueCheckOnlyInBatch: Boolean = DEFAULT_ENABLE_UNIQUE_CHECK_ONLY_WITHIN_BATCH,
1919
enableFastGeneration: Boolean = DEFAULT_ENABLE_FAST_GENERATION,
20-
enableForeignKeyV2: Boolean = DEFAULT_ENABLE_FOREIGN_KEY_V2
2120
)
2221

2322
case class FoldersConfig(

‎api/src/main/scala/io/github/datacatering/datacaterer/api/model/ConnectionDeserializer.scala‎

Lines changed: 0 additions & 37 deletions
This file was deleted.

‎api/src/main/scala/io/github/datacatering/datacaterer/api/model/Constants.scala‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
package io.github.datacatering.datacaterer.api.model
22

3-
import java.util.UUID
4-
53
object Constants {
64

75
lazy val PLAN_CLASS = "PLAN_CLASS"
@@ -232,10 +230,8 @@ object Constants {
232230
lazy val DEFAULT_ENABLE_VALIDATION = true
233231
lazy val DEFAULT_ENABLE_SUGGEST_VALIDATIONS = false
234232
lazy val DEFAULT_ENABLE_ALERTS = true
235-
lazy val DEFAULT_ENABLE_TRACK_ACTIVITY = true
236233
lazy val DEFAULT_ENABLE_UNIQUE_CHECK_ONLY_WITHIN_BATCH = false
237234
lazy val DEFAULT_ENABLE_FAST_GENERATION = false
238-
lazy val DEFAULT_ENABLE_FOREIGN_KEY_V2 = true
239235

240236
//folders defaults
241237
lazy val DEFAULT_PLAN_FILE_PATH = "/opt/app/plan/customer-create-plan.yaml"
@@ -631,7 +627,6 @@ object Constants {
631627
lazy val CONFIG_FLAGS_ALERTS = "enableAlerts"
632628
lazy val CONFIG_FLAGS_UNIQUE_CHECK_ONLY_IN_BATCH = "enableUniqueCheckOnlyInBatch"
633629
lazy val CONFIG_FLAGS_FAST_GENERATION = "enableFastGeneration"
634-
lazy val CONFIG_FLAGS_FOREIGN_KEY_V2 = "enableForeignKeyV2"
635630
//folder config
636631
lazy val CONFIG_FOLDER_PLAN_FILE_PATH = "planFilePath"
637632
lazy val CONFIG_FOLDER_TASK_FOLDER_PATH = "taskFolderPath"

‎api/src/main/scala/io/github/datacatering/datacaterer/api/model/PlanModels.scala‎

Lines changed: 5 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
package io.github.datacatering.datacaterer.api.model
22

3-
import com.fasterxml.jackson.annotation.JsonIgnoreProperties
43
import com.fasterxml.jackson.databind.annotation.JsonDeserialize
54
import io.github.datacatering.datacaterer.api.model.Constants.{DEFAULT_COUNT_RECORDS, DEFAULT_DATA_SOURCE_NAME, DEFAULT_FIELD_NAME, DEFAULT_FIELD_NULLABLE, DEFAULT_FIELD_TYPE, DEFAULT_PER_FIELD_COUNT_RECORDS, DEFAULT_PLAN_NAME, DEFAULT_STEP_ENABLED, DEFAULT_STEP_NAME, DEFAULT_STEP_TYPE, DEFAULT_TASK_NAME, DEFAULT_TASK_SUMMARY_ENABLE, FOREIGN_KEY_DELIMITER}
65

@@ -16,10 +15,7 @@ case class Plan(
1615
runId: Option[String] = Some(UUID.randomUUID().toString),
1716
runInterface: Option[String] = None,
1817
testType: Option[String] = None,
19-
testConfig: Option[TestConfig] = None,
20-
// New unified YAML format fields
21-
connections: Option[List[Connection]] = None,
22-
configuration: Option[Map[String, Any]] = None
18+
testConfig: Option[TestConfig] = None
2319
)
2420

2521
case class SinkOptions(
@@ -45,8 +41,7 @@ case class ForeignKeyRelation(
4541
case class ForeignKey(
4642
source: ForeignKeyRelation = ForeignKeyRelation(),
4743
generate: List[ForeignKeyRelation] = List(),
48-
delete: List[ForeignKeyRelation] = List(),
49-
relationshipType: Option[String] = None
44+
delete: List[ForeignKeyRelation] = List()
5045
)
5146

5247
/**
@@ -80,40 +75,14 @@ case class NullabilityConfig(
8075
require(nullPercentage >= 0.0 && nullPercentage <= 1.0, "nullPercentage must be between 0.0 and 1.0")
8176
}
8277

83-
/**
84-
* Many-to-many relationship configuration using junction table pattern.
85-
*
86-
* @param leftSource Left side of the relationship (e.g., students)
87-
* @param rightSource Right side of the relationship (e.g., courses)
88-
* @param junctionTable Junction/bridge table (e.g., enrollments)
89-
* @param leftCardinality Cardinality config for left side (e.g., courses per student)
90-
* @param rightCardinality Cardinality config for right side (e.g., students per course)
91-
*/
92-
case class ManyToManyRelation(
93-
leftSource: ForeignKeyRelation,
94-
rightSource: ForeignKeyRelation,
95-
junctionTable: ForeignKeyRelation,
96-
leftCardinality: Option[CardinalityConfig] = None,
97-
rightCardinality: Option[CardinalityConfig] = None
98-
) {
99-
}
100-
101-
@JsonIgnoreProperties(ignoreUnknown = true)
10278
case class TaskSummary(
10379
name: String,
104-
dataSourceName: String = "",
80+
dataSourceName: String,
10581
enabled: Boolean = DEFAULT_TASK_SUMMARY_ENABLE,
10682
weight: Option[Int] = None,
107-
stage: Option[String] = None,
108-
// Inline task definition fields (unified YAML format)
109-
steps: Option[List[Step]] = None,
110-
transformation: Option[TransformationConfig] = None,
111-
@JsonDeserialize(using = classOf[ConnectionDeserializer])
112-
connection: Option[Either[String, Connection]] = None,
113-
validations: Option[List[Any]] = None
83+
stage: Option[String] = None
11484
)
11585

116-
@JsonIgnoreProperties(ignoreUnknown = true)
11786
case class Task(
11887
name: String = DEFAULT_TASK_NAME,
11988
steps: List[Step] = List(),
@@ -127,9 +96,7 @@ case class Step(
12796
options: Map[String, String] = Map(),
12897
fields: List[Field] = List(),
12998
enabled: Boolean = DEFAULT_STEP_ENABLED,
130-
transformation: Option[TransformationConfig] = None,
131-
// New unified YAML format fields
132-
validations: Option[List[Any]] = None // Validation objects (field, expression, groupBy, metric)
99+
transformation: Option[TransformationConfig] = None
133100
)
134101

135102
case class Count(
@@ -184,18 +151,3 @@ case class LoadPatternStep(
184151
duration: String
185152
)
186153

187-
/**
188-
* Connection configuration for unified YAML format.
189-
* Can be used inline in tasks or defined in the connections section of a plan.
190-
*
191-
* @param name Optional connection name for reuse
192-
* @param `type` Connection type (postgres, csv, json, kafka, http, etc.)
193-
* @param options Connection options including url, credentials, format-specific settings, etc.
194-
* Common options: url, user, password, driver, path, header, delimiter, etc.
195-
*/
196-
case class Connection(
197-
name: Option[String] = None,
198-
`type`: String = "",
199-
options: Map[String, String] = Map()
200-
)
201-

0 commit comments

Comments
 (0)