Intg-1794: decode vins from file with option to persist (#292)

* implement option to read vins from file

* implement database persistance option

Author: JamesReate

cmd/device-definitions-api/decode_vin.go

@@ -2,8 +2,21 @@ package main
 
 import (
 	"context"
+	"encoding/csv"
 	"flag"
 	"fmt"
+	"io"
+	"os"
+
+	"github.com/DIMO-Network/device-definitions-api/internal/core/common"
+	coremodels "github.com/DIMO-Network/device-definitions-api/internal/core/models"
+	"github.com/DIMO-Network/device-definitions-api/internal/core/services"
+	"github.com/DIMO-Network/device-definitions-api/internal/infrastructure/db/models"
+	"github.com/DIMO-Network/shared/pkg/db"
+	vinutil "github.com/DIMO-Network/shared/pkg/vin"
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/volatiletech/null/v8"
+	"github.com/volatiletech/sqlboiler/v4/boil"
 
 	"github.com/goccy/go-json"
 
@@ -17,86 +30,233 @@ type decodeVINCmd struct {
 	logger   *zerolog.Logger
 	settings *config.Settings
 
-	datGroup   bool
-	drivly     bool
-	vincario   bool
-	japan17vin bool
+	datGroup    bool
+	drivly      bool
+	vincario    bool
+	japan17vin  bool
+	fromFile    bool
+	persistToDB bool
 }
 
 func (*decodeVINCmd) Name() string { return "decodevin" }
 func (*decodeVINCmd) Synopsis() string {
 	return "tries decoding a vin with chosen provider - does not insert in our db"
 }
 func (*decodeVINCmd) Usage() string {
-	return `decodevin [-dat|-drivly|-vincario|-japan17vin] <vin 17 chars> <country two letter iso>`
+	return `decodevin [-dat|-drivly|-vincario|-japan17vin|-from-file] <vin 17 chars OR filaname in /tmp> <country two letter iso>`
 }
 
 func (p *decodeVINCmd) SetFlags(f *flag.FlagSet) {
 	f.BoolVar(&p.datGroup, "dat", false, "use dat group vin decoder")
 	f.BoolVar(&p.drivly, "drivly", false, "use drivly vin decoder")
 	f.BoolVar(&p.vincario, "vincario", false, "use vincario vin decoder")
 	f.BoolVar(&p.japan17vin, "japan17vin", false, "use japan17vin vin decoder")
+	f.BoolVar(&p.fromFile, "from-file", false, "read vin from file in /tmp directory")
+	f.BoolVar(&p.persistToDB, "persist-to-db", false, "persist successful vin decodings to db, table vin_numbers")
 }
 
-func (p *decodeVINCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
+func (p *decodeVINCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
 	if len(f.Args()) == 0 {
-		fmt.Println("missing vin parameter")
+		if p.fromFile {
+			fmt.Println("missing filename parameter")
+		} else {
+			fmt.Println("missing vin parameter")
+		}
 		return subcommands.ExitUsageError
 	}
-	vin := f.Args()[0]
+	vinOrFile := f.Args()[0]
+
+	pdb := db.NewDbConnectionFromSettings(context.Background(), &p.settings.DB, true)
+	pdb.WaitForDB(*p.logger)
 
 	country := "USA"
 	if len(f.Args()) == 2 {
 		country = f.Args()[1]
 	}
-	fmt.Printf("VIN: %s\n", vin)
+	vins := []string{}
+	if p.fromFile {
+		fmt.Printf("Filename: %s\n", vinOrFile)
+		vins = loadVINsFromFile(vinOrFile)
+	} else {
+		fmt.Printf("VIN: %s\n", vinOrFile)
+		vins = append(vins, vinOrFile)
+	}
 	fmt.Printf("Country: %s\n", country)
 
-	if p.datGroup {
-		// use the dat group service to decode
-		datAPI := gateways.NewDATGroupAPIService(p.settings, p.logger)
-		vinInfo, err := datAPI.GetVINv2(vin, country)
+	fmt.Printf("total VINs found: %d\n", len(vins))
+	if len(vins) == 0 {
+		fmt.Println("no vins found")
+		return subcommands.ExitFailure
+	}
+
+	vinDecodingService := instantiateVINDecodingSvc(ctx, p.settings, p.logger, pdb)
+
+	for _, vin := range vins {
+		// in case want to insert
+		vinObj := vinutil.VIN(vin)
+		dbVin := &models.VinNumber{
+			Vin:          vin,
+			Wmi:          null.StringFrom(vinObj.Wmi()),
+			VDS:          null.StringFrom(vinObj.VDS()),
+			SerialNumber: vinObj.SerialNumber(),
+			CheckDigit:   null.StringFrom(vinObj.CheckDigit()),
+			Vis:          null.StringFrom(vinObj.VIS()),
+		}
+		wmi, _ := models.Wmis(models.WmiWhere.Wmi.EQ(vinObj.Wmi())).One(ctx, pdb.DBS().Reader)
+		if wmi != nil {
+			dbVin.ManufacturerName = wmi.ManufacturerName
+		}
+		dt, err := models.DeviceTypes(models.DeviceTypeWhere.ID.EQ(common.DefaultDeviceType)).One(ctx, pdb.DBS().Reader)
 		if err != nil {
 			fmt.Println(err.Error())
 			return subcommands.ExitFailure
 		}
+		vinInfo := &coremodels.VINDecodingInfoData{VIN: vin}
 
-		fmt.Printf("\n\nVIN Response: %+v\n", *vinInfo)
-	}
-	if p.drivly {
-		drivlyAPI := gateways.NewDrivlyAPIService(p.settings)
-		vinInfo, err := drivlyAPI.GetVINInfo(vin)
-		if err != nil {
-			fmt.Println(err.Error())
-			return subcommands.ExitFailure
+		if p.datGroup {
+			vinInfo, err = vinDecodingService.GetVIN(ctx, vin, dt, coremodels.DATGroupProvider, country)
+			// use the dat group service to decode
+			if err != nil {
+				fmt.Println(err.Error())
+			}
+
+			fmt.Printf("\n\nVIN Response: %+v\n", vinInfo)
+		}
+		if p.drivly {
+			vinInfo, err = vinDecodingService.GetVIN(ctx, vin, dt, coremodels.DrivlyProvider, country)
+			if err != nil {
+				fmt.Println(err.Error())
+				return subcommands.ExitFailure
+			}
+
+			fmt.Printf("VIN Response: %+v\n", vinInfo)
 		}
+		if p.vincario {
+			vinInfo, err = vinDecodingService.GetVIN(ctx, vin, dt, coremodels.VincarioProvider, country)
+			if err != nil {
+				fmt.Println(err.Error())
+				return subcommands.ExitFailure
+			}
 
-		fmt.Printf("VIN Response: %+v\n", vinInfo)
+			fmt.Printf("VIN Response: %+v\n", vinInfo)
+		}
+		if p.japan17vin {
+			vinInfo, err = vinDecodingService.GetVIN(ctx, vin, dt, coremodels.Japan17VIN, country)
+			if err != nil {
+				fmt.Println(err.Error())
+				return subcommands.ExitFailure
+			}
+			jsonBytes, _ := json.MarshalIndent(vinInfo, "", " ")
+			fmt.Println("VIN Info:")
+			fmt.Println(string(jsonBytes))
+		}
+		fmt.Println()
+		if p.persistToDB {
+			if vinInfo == nil || vinInfo.Model == "" {
+				fmt.Println("no decoding info found, skipping: " + vin)
+				continue
+			}
+			dbVin.Year = int(vinInfo.Year)
+			if dbVin.ManufacturerName == "" {
+				dbVin.ManufacturerName = vinInfo.Make
+			}
+			dbVin.DatgroupData = null.JSONFrom(vinInfo.Raw)
+			dbVin.DefinitionID = common.DeviceDefinitionSlug(vinInfo.Make, vinInfo.Model, int16(vinInfo.Year))
+			dbVin.DecodeProvider = null.StringFrom(string(vinInfo.Source))
+			// todo future change to add field with StyleName
+
+			err := dbVin.Insert(ctx, pdb.DBS().Writer, boil.Infer())
+			if err != nil {
+				fmt.Println(err.Error())
+				return subcommands.ExitFailure
+			}
+		}
 	}
-	if p.vincario {
-		vincarioAPI := gateways.NewVincarioAPIService(p.settings, p.logger)
-		vinInfo, err := vincarioAPI.DecodeVIN(vin)
-		if err != nil {
-			fmt.Println(err.Error())
-			return subcommands.ExitFailure
+	return subcommands.ExitSuccess
+}
+
+func loadVINsFromFile(file string) []string {
+	// files are assumed to be in the tmp directory
+	// pull out vins from csv file from column "vin"
+	vinFile := "/tmp/" + file
+	vinFileContents, err := readVINFile(vinFile)
+	if err != nil {
+		fmt.Println(err.Error())
+		return []string{}
+	}
+	return vinFileContents
+}
+
+func readVINFile(filename string) ([]string, error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open file: %w", err)
+	}
+	defer file.Close()
+
+	reader := csv.NewReader(file)
+
+	// Read header row to find the index of "csv" column
+	header, err := reader.Read()
+	if err != nil {
+		return nil, fmt.Errorf("failed to read CSV header: %w", err)
+	}
+
+	csvColumnIndex := -1
+	for i, columnName := range header {
+		if columnName == "csv" {
+			csvColumnIndex = i
+			break
 		}
+	}
 
-		fmt.Printf("VIN Response: %+v\n", vinInfo)
+	if csvColumnIndex == -1 {
+		csvColumnIndex = 0 // default to first column if "csv" column not found
+		fmt.Println("defaulting to first column as 'csv' column not found, please ensure your CSV file has a column named 'csv' with VINs in it.")
 	}
-	if p.japan17vin {
-		jp17vinAPI := gateways.NewJapan17VINAPI(p.logger, p.settings)
-		vinInfo, payload, err := jp17vinAPI.GetVINInfo(vin)
+
+	// Read all rows and extract values from the "csv" column
+	var values []string
+	for {
+		row, err := reader.Read()
+		if err == io.EOF {
+			break
+		}
 		if err != nil {
-			fmt.Println(err.Error())
-			return subcommands.ExitFailure
+			return nil, fmt.Errorf("failed to read CSV row: %w", err)
+		}
+
+		if csvColumnIndex < len(row) {
+			values = append(values, row[csvColumnIndex])
 		}
-		jsonBytes, _ := json.MarshalIndent(vinInfo, "", " ")
-		fmt.Println("VIN Info:")
-		fmt.Println(string(jsonBytes))
-		fmt.Println("Raw JSON Payload:")
-		fmt.Println(string(payload))
 	}
 
-	fmt.Println()
-	return subcommands.ExitSuccess
+	return values, nil
+}
+
+func instantiateVINDecodingSvc(ctx context.Context, settings *config.Settings, logger *zerolog.Logger, pdb db.Store) services.VINDecodingService {
+	datAPI := gateways.NewDATGroupAPIService(settings, logger)
+	drivlyAPI := gateways.NewDrivlyAPIService(settings)
+	vincarioAPI := gateways.NewVincarioAPIService(settings, logger)
+	jp17vinAPI := gateways.NewJapan17VINAPI(logger, settings)
+
+	send, err := createSender(ctx, settings, logger)
+	if err != nil {
+		logger.Fatal().Err(err).Msg("Failed to create sender.")
+	}
+
+	ethClient, err := ethclient.Dial(settings.EthereumRPCURL.String())
+	if err != nil {
+		logger.Fatal().Err(err).Msg("Failed to create Ethereum client.")
+	}
+
+	chainID, err := ethClient.ChainID(ctx)
+	if err != nil {
+		logger.Fatal().Err(err).Msg("Couldn't retrieve chain id.")
+	}
+	deviceDefinitionOnChainService := gateways.NewDeviceDefinitionOnChainService(settings, logger, ethClient, chainID, send, pdb.DBS)
+
+	vinDecodingService := services.NewVINDecodingService(drivlyAPI, vincarioAPI, nil, logger, deviceDefinitionOnChainService, datAPI, pdb.DBS, jp17vinAPI)
+
+	return vinDecodingService
 }