Skip to content

Commit 6af5dc1

Browse files
sholokhovsholokhov
committed
move core logic into lib
1 parent f3078ba commit 6af5dc1

15 files changed

Lines changed: 47 additions & 587 deletions

‎cmd/check.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,16 @@ package cmd
22

33
import (
44
"fmt"
5+
"github.com/DataBridgeTech/dbqcore"
56
"log"
67
"os"
78
"strings"
89

9-
"github.com/DataBridge-Tech/dbq/internal"
10+
"github.com/DataBridgeTech/dbq/internal"
1011
"github.com/spf13/cobra"
1112
)
1213

13-
func NewCheckCommand(app internal.DbqApp) *cobra.Command {
14+
func NewCheckCommand(app internal.DbqCliApp) *cobra.Command {
1415
var checksFile string
1516

1617
cmd := &cobra.Command{
@@ -24,7 +25,7 @@ By automating these checks, you can proactively identify and address data qualit
2425
RunE: func(cmd *cobra.Command, args []string) error {
2526
log.Printf("Reading checks configuration file: %s \n", checksFile)
2627

27-
checksCfg, err := internal.LoadChecksConfig(checksFile)
28+
checksCfg, err := dbqcore.LoadChecksConfig(checksFile)
2829
if err != nil {
2930
return fmt.Errorf("error while loading checks configuration file: %w", err)
3031
}
@@ -50,7 +51,7 @@ By automating these checks, you can proactively identify and address data qualit
5051
}
5152

5253
log.Printf(" [%d/%d] '%s': %s", cIdx+1, len(rule.Checks), check.ID, getCheckResultLabel(pass))
53-
if !pass && internal.IdOrDefault(string(check.OnFail), internal.OnFailActionError) == internal.OnFailActionError {
54+
if !pass && internal.IdOrDefault(string(check.OnFail), dbqcore.OnFailActionError) == dbqcore.OnFailActionError {
5455
exitCode = 1
5556
}
5657
}

‎cmd/import.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@ package cmd
33
import (
44
"log"
55

6-
"github.com/DataBridge-Tech/dbq/internal"
6+
"github.com/DataBridgeTech/dbq/internal"
77
"github.com/spf13/cobra"
88
)
99

10-
func NewImportCommand(app internal.DbqApp) *cobra.Command {
10+
func NewImportCommand(app internal.DbqCliApp) *cobra.Command {
1111
var dataSource string
1212
var filter string
1313
var updateCfg bool

‎cmd/ping.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@ package cmd
33
import (
44
"log"
55

6-
"github.com/DataBridge-Tech/dbq/internal"
6+
"github.com/DataBridgeTech/dbq/internal"
77
"github.com/spf13/cobra"
88
)
99

10-
func NewPingCommand(app internal.DbqApp) *cobra.Command {
10+
func NewPingCommand(app internal.DbqCliApp) *cobra.Command {
1111
var dataSource string
1212

1313
cmd := &cobra.Command{

‎cmd/profile.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,14 @@ package cmd
22

33
import (
44
"encoding/json"
5+
"github.com/DataBridgeTech/dbqcore"
56
"log"
67

7-
"github.com/DataBridge-Tech/dbq/internal"
8+
"github.com/DataBridgeTech/dbq/internal"
89
"github.com/spf13/cobra"
910
)
1011

11-
func NewProfileCommand(app internal.DbqApp) *cobra.Command {
12+
func NewProfileCommand(app internal.DbqCliApp) *cobra.Command {
1213
var dataSource string
1314
var dataSet string
1415
var sample bool
@@ -36,8 +37,8 @@ and helps in making better decisions about data processing and analysis.
3637
}
3738
}
3839

39-
profileResults := &internal.ProfileResultOutput{
40-
Profiles: make(map[string]*internal.TableMetrics),
40+
profileResults := &dbqcore.ProfileResultOutput{
41+
Profiles: make(map[string]*dbqcore.TableMetrics),
4142
}
4243

4344
for _, curDataSet := range dataSetsToProfile {

‎cmd/root.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import (
44
"log/slog"
55
"os"
66

7-
"github.com/DataBridge-Tech/dbq/internal"
7+
"github.com/DataBridgeTech/dbq/internal"
88
"github.com/spf13/cobra"
99
)
1010

@@ -22,7 +22,7 @@ func Execute() {
2222
}
2323
}
2424

25-
func AddCommands(app internal.DbqApp) {
25+
func AddCommands(app internal.DbqCliApp) {
2626
rootCmd.AddCommand(NewPingCommand(app))
2727
rootCmd.AddCommand(NewImportCommand(app))
2828
rootCmd.AddCommand(NewCheckCommand(app))

‎cmd/version.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ package cmd
33
import (
44
"fmt"
55

6-
dbqcore "github.com/DataBridge-Tech/dbq-core"
6+
"github.com/DataBridgeTech/dbqcore"
77
"github.com/spf13/cobra"
88
)
99

‎go.mod‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,16 @@
1-
module github.com/DataBridge-Tech/dbq
1+
module github.com/DataBridgeTech/dbq
22

33
go 1.24.2
44

55
require (
66
github.com/ClickHouse/clickhouse-go/v2 v2.34.0
7+
github.com/DataBridgeTech/dbqcore v0.0.1
78
github.com/spf13/cobra v1.9.1
89
github.com/spf13/viper v1.20.1
910
gopkg.in/yaml.v3 v3.0.1
10-
github.com/DataBridge-Tech/dbq-core v0.0.1
1111
)
1212

13-
// replace github.com/DataBridge-Tech/dbq-core => ../dbq-core
13+
//replace github.com/DataBridgeTech/dbqcore => ../dbqcore
1414

1515
require (
1616
github.com/ClickHouse/ch-go v0.65.1 // indirect
@@ -21,7 +21,7 @@ require (
2121
github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
2222
github.com/google/uuid v1.6.0 // indirect
2323
github.com/inconshreveable/mousetrap v1.1.0 // indirect
24-
github.com/klauspost/compress v1.17.11 // indirect
24+
github.com/klauspost/compress v1.18.0 // indirect
2525
github.com/paulmach/orb v0.11.1 // indirect
2626
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
2727
github.com/pierrec/lz4/v4 v4.1.22 // indirect
@@ -36,6 +36,6 @@ require (
3636
go.opentelemetry.io/otel v1.35.0 // indirect
3737
go.opentelemetry.io/otel/trace v1.35.0 // indirect
3838
go.uber.org/multierr v1.11.0 // indirect
39-
golang.org/x/sys v0.30.0 // indirect
39+
golang.org/x/sys v0.33.0 // indirect
4040
golang.org/x/text v0.21.0 // indirect
4141
)

‎go.sum‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
3434
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
3535
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
3636
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
37+
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
38+
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
3739
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
3840
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
3941
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@@ -118,6 +120,8 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
118120
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
119121
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
120122
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
123+
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
124+
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
121125
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
122126
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
123127
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

‎internal/app.go‎

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -6,29 +6,31 @@ import (
66
"os"
77
"strings"
88

9+
"github.com/DataBridgeTech/dbqcore"
10+
911
"github.com/spf13/cobra"
1012
"github.com/spf13/viper"
1113
"gopkg.in/yaml.v3"
1214
)
1315

14-
type DbqApp interface {
16+
type DbqCliApp interface {
1517
PingDataSource(srcId string) (string, error)
1618
ImportDatasets(srcId string, filter string) ([]string, error)
17-
ProfileDataset(srcId string, dataset string, sample bool) (*TableMetrics, error)
18-
GetDbqConfig() *DbqConfig
19+
ProfileDataset(srcId string, dataset string, sample bool) (*dbqcore.TableMetrics, error)
20+
GetDbqConfig() *dbqcore.DbqConfig
1921
SaveDbqConfig() error
20-
FindDataSourceById(srcId string) *DataSource
21-
RunCheck(check *Check, dataSource *DataSource, dataset string, defaultWhere string) (bool, string, error)
22+
FindDataSourceById(srcId string) *dbqcore.DataSource
23+
RunCheck(check *dbqcore.Check, dataSource *dbqcore.DataSource, dataset string, defaultWhere string) (bool, string, error)
2224
SetLogLevel(level slog.Level)
2325
}
2426

2527
type DbqAppImpl struct {
2628
dbqConfigPath string
27-
dbqConfig *DbqConfig
29+
dbqConfig *dbqcore.DbqConfig
2830
logLevel slog.Level
2931
}
3032

31-
func NewDbqApp(dbqConfigPath string) DbqApp {
33+
func NewDbqCliApp(dbqConfigPath string) DbqCliApp {
3234
dbqConfig, dbqConfigUsedPath := initConfig(dbqConfigPath)
3335
return &DbqAppImpl{
3436
dbqConfigPath: dbqConfigUsedPath,
@@ -63,7 +65,7 @@ func (app *DbqAppImpl) ImportDatasets(srcId string, filter string) ([]string, er
6365
return cnn.ImportDatasets(filter)
6466
}
6567

66-
func (app *DbqAppImpl) ProfileDataset(srcId string, dataset string, sample bool) (*TableMetrics, error) {
68+
func (app *DbqAppImpl) ProfileDataset(srcId string, dataset string, sample bool) (*dbqcore.TableMetrics, error) {
6769
var dataSource = app.FindDataSourceById(srcId)
6870
cnn, err := getDbqConnector(*dataSource)
6971
if err != nil {
@@ -72,7 +74,7 @@ func (app *DbqAppImpl) ProfileDataset(srcId string, dataset string, sample bool)
7274
return cnn.ProfileDataset(dataset, sample)
7375
}
7476

75-
func (app *DbqAppImpl) GetDbqConfig() *DbqConfig {
77+
func (app *DbqAppImpl) GetDbqConfig() *dbqcore.DbqConfig {
7678
return app.dbqConfig
7779
}
7880

@@ -90,7 +92,7 @@ func (app *DbqAppImpl) SaveDbqConfig() error {
9092
return nil
9193
}
9294

93-
func (app *DbqAppImpl) FindDataSourceById(srcId string) *DataSource {
95+
func (app *DbqAppImpl) FindDataSourceById(srcId string) *dbqcore.DataSource {
9496
for i := range app.dbqConfig.DataSources {
9597
if app.dbqConfig.DataSources[i].ID == srcId {
9698
return &app.dbqConfig.DataSources[i]
@@ -99,7 +101,7 @@ func (app *DbqAppImpl) FindDataSourceById(srcId string) *DataSource {
99101
return nil
100102
}
101103

102-
func (app *DbqAppImpl) RunCheck(check *Check, dataSource *DataSource, dataset string, defaultWhere string) (bool, string, error) {
104+
func (app *DbqAppImpl) RunCheck(check *dbqcore.Check, dataSource *dbqcore.DataSource, dataset string, defaultWhere string) (bool, string, error) {
103105
cnn, err := getDbqConnector(*dataSource)
104106
if err != nil {
105107
return false, "", err
@@ -111,7 +113,7 @@ func (app *DbqAppImpl) SetLogLevel(logLevel slog.Level) {
111113
app.logLevel = logLevel
112114
}
113115

114-
func initConfig(dbqConfigPath string) (*DbqConfig, string) {
116+
func initConfig(dbqConfigPath string) (*dbqcore.DbqConfig, string) {
115117
v := viper.New()
116118

117119
if dbqConfigPath != "" {
@@ -129,19 +131,19 @@ func initConfig(dbqConfigPath string) (*DbqConfig, string) {
129131
cobra.CheckErr(err)
130132
}
131133

132-
var dbqConfig DbqConfig
134+
var dbqConfig dbqcore.DbqConfig
133135
if err := v.Unmarshal(&dbqConfig); err != nil {
134136
cobra.CheckErr(err)
135137
}
136138

137139
return &dbqConfig, v.ConfigFileUsed()
138140
}
139141

140-
func getDbqConnector(ds DataSource) (DbqConnector, error) {
142+
func getDbqConnector(ds dbqcore.DataSource) (dbqcore.DbqConnector, error) {
141143
dsType := strings.ToLower(ds.Type)
142144
switch dsType {
143145
case "clickhouse":
144-
return NewClickhouseDbqConnector(ds)
146+
return dbqcore.NewClickhouseDbqConnector(ds)
145147
default:
146148
return nil, fmt.Errorf("data source type '%s' is not supported", dsType)
147149
}

‎internal/checks_config.go‎

Lines changed: 0 additions & 48 deletions
This file was deleted.

0 commit comments

Comments
 (0)