Skip to content

Commit 1473000

Browse files
JamesReateJamesReate
committed
script to load bulk vins with csv
1 parent eaca1b8 commit 1473000

3 files changed

Lines changed: 220 additions & 0 deletions

File tree

‎cmd/device-definitions-api/add_vins_csv.go‎

Lines changed: 215 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,215 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"database/sql"
6+
"encoding/csv"
7+
"flag"
8+
"fmt"
9+
"io"
10+
"math/big"
11+
"os"
12+
"strings"
13+
14+
"github.com/DIMO-Network/device-definitions-api/internal/config"
15+
"github.com/DIMO-Network/device-definitions-api/internal/infrastructure/db/models"
16+
"github.com/DIMO-Network/device-definitions-api/internal/infrastructure/gateways"
17+
"github.com/DIMO-Network/device-definitions-api/internal/infrastructure/sender"
18+
"github.com/DIMO-Network/shared/pkg/db"
19+
stringutils "github.com/DIMO-Network/shared/pkg/strings"
20+
vinutils "github.com/DIMO-Network/shared/pkg/vin"
21+
"github.com/aarondl/null/v8"
22+
"github.com/aarondl/sqlboiler/v4/boil"
23+
"github.com/ethereum/go-ethereum/ethclient"
24+
"github.com/google/subcommands"
25+
"github.com/pkg/errors"
26+
"github.com/rs/zerolog"
27+
)
28+
29+
type addVINsCSVCmd struct {
30+
logger zerolog.Logger
31+
settings config.Settings
32+
33+
sender sender.Sender
34+
}
35+
36+
func (*addVINsCSVCmd) Name() string { return "addvinscsv" }
37+
func (*addVINsCSVCmd) Synopsis() string {
38+
return "bulk adds VINs from CSV text with VIN and DefinitionId columns"
39+
}
40+
func (*addVINsCSVCmd) Usage() string {
41+
return `addvinscsv:
42+
Reads CSV text from stdin with columns: VIN, DefinitionId
43+
Example:
44+
cat vins.csv | go run . addvinscsv
45+
Or:
46+
echo "VIN,DefinitionId
47+
1HGBH41JXMN109186,some-definition-id
48+
2HGFC2F59FH123456,another-definition-id" | go run . addvinscsv
49+
`
50+
}
51+
52+
func (p *addVINsCSVCmd) SetFlags(_ *flag.FlagSet) {
53+
}
54+
55+
func (p *addVINsCSVCmd) Execute(ctx context.Context, _ *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
56+
pdb := db.NewDbConnectionFromSettings(ctx, &p.settings.DB, true)
57+
pdb.WaitForDB(p.logger)
58+
59+
ethClient, err := ethclient.Dial(p.settings.EthereumRPCURL.String())
60+
if err != nil {
61+
p.logger.Fatal().Err(err).Msg("Failed to create Ethereum client.")
62+
}
63+
64+
chainID, err := ethClient.ChainID(ctx)
65+
if err != nil {
66+
p.logger.Fatal().Err(err).Msg("Couldn't retrieve chain id.")
67+
}
68+
onChainSvc := gateways.NewDeviceDefinitionOnChainService(&p.settings, &p.logger, ethClient, chainID, p.sender, pdb.DBS)
69+
70+
// Read CSV from stdin
71+
reader := csv.NewReader(os.Stdin)
72+
reader.TrimLeadingSpace = true
73+
74+
// Read header row
75+
headers, err := reader.Read()
76+
if err != nil {
77+
p.logger.Error().Err(err).Msg("Failed to read CSV headers")
78+
fmt.Println("Failed to read CSV headers:", err)
79+
return subcommands.ExitFailure
80+
}
81+
82+
// Find column indices
83+
vinIdx := -1
84+
defIDIdx := -1
85+
for i, header := range headers {
86+
headerLower := strings.ToLower(strings.TrimSpace(header))
87+
switch headerLower {
88+
case "vin":
89+
vinIdx = i
90+
case "definitionid":
91+
defIDIdx = i
92+
}
93+
}
94+
95+
if vinIdx == -1 || defIDIdx == -1 {
96+
p.logger.Error().Msg("CSV must contain 'VIN' and 'DefinitionId' columns")
97+
fmt.Println("Error: CSV must contain 'VIN' and 'DefinitionId' columns")
98+
return subcommands.ExitFailure
99+
}
100+
101+
successCount := 0
102+
skipCount := 0
103+
errorCount := 0
104+
105+
// Process each row
106+
for {
107+
record, err := reader.Read()
108+
if err == io.EOF {
109+
break
110+
}
111+
if err != nil {
112+
p.logger.Error().Err(err).Msg("Failed to read CSV row")
113+
fmt.Println("Failed to read CSV row:", err)
114+
errorCount++
115+
continue
116+
}
117+
118+
if len(record) <= vinIdx || len(record) <= defIDIdx {
119+
p.logger.Error().Msg("Invalid CSV row: missing columns")
120+
fmt.Println("Invalid CSV row: missing columns")
121+
errorCount++
122+
continue
123+
}
124+
125+
vin := strings.TrimSpace(record[vinIdx])
126+
definitionID := strings.TrimSpace(record[defIDIdx])
127+
128+
if len(vin) != 17 {
129+
p.logger.Error().Str("vin", vin).Msg("Invalid VIN: must be 17 characters")
130+
fmt.Printf("Skipping invalid VIN '%s': must be 17 characters\n", vin)
131+
errorCount++
132+
continue
133+
}
134+
135+
// Check if VIN already exists
136+
vinDecodeNumber, err := models.FindVinNumber(ctx, pdb.DBS().Reader, vin)
137+
if err != nil && !errors.Is(err, sql.ErrNoRows) {
138+
p.logger.Error().Err(err).Str("vin", vin).Msg("Database error checking VIN")
139+
fmt.Printf("Error checking VIN '%s': %v\n", vin, err)
140+
errorCount++
141+
continue
142+
}
143+
if vinDecodeNumber != nil {
144+
p.logger.Info().Str("vin", vin).Msg("VIN already registered, skipping")
145+
fmt.Printf("VIN '%s' already registered, skipping\n", vin)
146+
skipCount++
147+
continue
148+
}
149+
150+
// Process VIN
151+
processedVIN := vinutils.VIN(vin)
152+
wmi, err := models.Wmis(models.WmiWhere.Wmi.EQ(processedVIN.Wmi())).One(ctx, pdb.DBS().Reader)
153+
if err != nil {
154+
p.logger.Error().Err(err).Str("vin", vin).Str("wmi", processedVIN.Wmi()).Msg("Could not find WMI for VIN")
155+
fmt.Printf("Error: Could not find WMI '%s' for VIN '%s'\n", processedVIN.Wmi(), vin)
156+
errorCount++
157+
continue
158+
}
159+
// Verify the device definition exists
160+
manufacturer, err := onChainSvc.GetManufacturer(stringutils.SlugString(wmi.ManufacturerName))
161+
if err != nil {
162+
p.logger.Error().Err(err).Str("vin", vin).Str("manufacturer", wmi.ManufacturerName).Msg("Could not find manufacturer")
163+
fmt.Printf("Error: Could not find manufacturer '%s' for VIN '%s': %v\n", wmi.ManufacturerName, vin, err)
164+
errorCount++
165+
continue
166+
}
167+
168+
deviceDefinition, err := onChainSvc.GetDefinitionTableland(ctx, big.NewInt(int64(manufacturer.TokenID)), definitionID)
169+
if err != nil {
170+
p.logger.Error().Err(err).Str("vin", vin).Str("definitionID", definitionID).Msg("Could not find device definition")
171+
fmt.Printf("Error: Could not find device definition '%s' for VIN '%s': %v\n", definitionID, vin, err)
172+
errorCount++
173+
continue
174+
}
175+
176+
vinNumber := models.VinNumber{
177+
Vin: vin,
178+
Wmi: null.StringFrom(processedVIN.Wmi()),
179+
VDS: null.StringFrom(processedVIN.VDS()),
180+
SerialNumber: processedVIN.SerialNumber(),
181+
CheckDigit: null.StringFrom(processedVIN.CheckDigit()),
182+
Vis: null.StringFrom(processedVIN.VIS()),
183+
ManufacturerName: wmi.ManufacturerName,
184+
DecodeProvider: null.StringFrom("csv-import"),
185+
DefinitionID: definitionID,
186+
Year: deviceDefinition.Year,
187+
}
188+
189+
// Insert into database
190+
err = vinNumber.Insert(ctx, pdb.DBS().Writer, boil.Infer())
191+
if err != nil {
192+
p.logger.Error().Err(err).Str("vin", vin).Msg("Failed to insert VIN")
193+
fmt.Printf("Error inserting VIN '%s': %v\n", vin, err)
194+
errorCount++
195+
continue
196+
}
197+
198+
p.logger.Info().Str("vin", vin).Str("definitionID", vinNumber.DefinitionID).Msg("Successfully added VIN")
199+
fmt.Printf("✓ Added VIN '%s' with definition ID '%s'\n", vin, vinNumber.DefinitionID)
200+
successCount++
201+
}
202+
203+
// Print summary
204+
fmt.Println("\n=== Summary ===")
205+
fmt.Printf("Successfully added: %d\n", successCount)
206+
fmt.Printf("Skipped (already exist): %d\n", skipCount)
207+
fmt.Printf("Errors: %d\n", errorCount)
208+
fmt.Printf("Total processed: %d\n", successCount+skipCount+errorCount)
209+
210+
if errorCount > 0 {
211+
return subcommands.ExitFailure
212+
}
213+
214+
return subcommands.ExitSuccess
215+
}

‎cmd/device-definitions-api/main.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ func main() {
4949
subcommands.Register(subcommands.CommandsCommand(), "")
5050
subcommands.Register(&migrateDBCmd{logger: logger, settings: settings}, "")
5151
subcommands.Register(&addVINCmd{logger: logger, settings: settings}, "")
52+
subcommands.Register(&addVINsCSVCmd{logger: logger, settings: settings, sender: sigSender}, "")
5253
subcommands.Register(&decodeVINCmd{logger: &logger, settings: &settings}, "")
5354
subcommands.Register(&syncDeviceDefinitionSearchCmd{logger: logger, settings: settings, sender: sigSender}, "")
5455
subcommands.Register(&deleteDefinition{logger: logger, settings: settings}, "")

‎sample_vins.csv‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
VIN,DefinitionId
2+
1HGBH41JXMN109186,honda_accord_2008
3+
2HGFC2F59FH123456,honda_civic_2015
4+

0 commit comments

Comments
 (0)