Skip to content

Commit 759a186

Browse files
nickmangonickmango
committed
[#4468] Bug/Ecla Records
- Script that fixes invalid dates for ecla records Signed-off-by: Harold Wanyama <hwanyama@contractor.linuxfoundation.org>
1 parent 373849c commit 759a186

3 files changed

Lines changed: 297 additions & 0 deletions

File tree

‎cla-backend-go/cmd/migrate_ecla/main.go‎

Lines changed: 237 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,237 @@
1+
// Copyright The Linux Foundation and each contributor to CommunityBridge.
2+
// SPDX-License-Identifier: MIT
3+
4+
package main
5+
6+
import (
7+
"context"
8+
// "fmt"
9+
"os"
10+
"sync"
11+
"flag"
12+
13+
// "sync"
14+
15+
"github.com/aws/aws-sdk-go/aws"
16+
"github.com/aws/aws-sdk-go/aws/session"
17+
"github.com/communitybridge/easycla/cla-backend-go/company"
18+
19+
// "github.com/aws/aws-sdk-go/service/dynamodb/expression"
20+
"github.com/communitybridge/easycla/cla-backend-go/events"
21+
"github.com/communitybridge/easycla/cla-backend-go/gen/v1/models"
22+
eventOps "github.com/communitybridge/easycla/cla-backend-go/gen/v1/restapi/operations/events"
23+
log "github.com/communitybridge/easycla/cla-backend-go/logging"
24+
"github.com/communitybridge/easycla/cla-backend-go/project/repository"
25+
"github.com/communitybridge/easycla/cla-backend-go/projects_cla_groups"
26+
"github.com/communitybridge/easycla/cla-backend-go/repositories"
27+
"github.com/communitybridge/easycla/cla-backend-go/signatures"
28+
"github.com/communitybridge/easycla/cla-backend-go/users"
29+
"github.com/sirupsen/logrus"
30+
)
31+
32+
var stage string
33+
var eventsRepo events.Repository
34+
var eventsService events.Service
35+
var signatureRepo signatures.SignatureRepository
36+
var v1ProjectRepo repository.ProjectRepository
37+
var usersRepo users.UserRepository
38+
var companyRepo company.IRepository
39+
var ghRepo repositories.Repository
40+
var projectClaGroup projects_cla_groups.Repository
41+
var awsSession = session.Must(session.NewSession(&aws.Config{}))
42+
43+
type combinedRepo struct {
44+
users.UserRepository
45+
company.IRepository
46+
repository.ProjectRepository
47+
projects_cla_groups.Repository
48+
}
49+
50+
func init() {
51+
stage = os.Getenv("STAGE")
52+
if stage == "" {
53+
log.Fatal("STAGE environment variable not set")
54+
}
55+
56+
log.Infof("STAGE: %s", stage)
57+
companyRepo = company.NewRepository(awsSession, stage)
58+
usersRepo = users.NewRepository(awsSession, stage)
59+
ghRepo = *repositories.NewRepository(awsSession, stage)
60+
projectClaGroup = projects_cla_groups.NewRepository(awsSession, stage)
61+
v1ProjectRepo = repository.NewRepository(awsSession, stage, &ghRepo, nil, projectClaGroup)
62+
eventsRepo = events.NewRepository(awsSession, stage)
63+
eventsService = events.NewService(eventsRepo, combinedRepo{
64+
usersRepo,
65+
companyRepo,
66+
v1ProjectRepo,
67+
projectClaGroup,
68+
})
69+
signatureRepo = signatures.NewRepository(awsSession, stage, companyRepo, usersRepo, eventsService, &ghRepo, nil, nil, nil)
70+
71+
log.Infof("initializing migrate_ecla")
72+
}
73+
74+
func main() {
75+
f := logrus.Fields{
76+
"functionName": "main",
77+
"stage": stage,
78+
}
79+
80+
dryRun := flag.Bool("dry-run",true, "boolean flag that can update the signature records")
81+
flag.Parse()
82+
log.WithFields(f).Info("migrate_ecla started")
83+
84+
ctx := context.Background()
85+
// Fetch all the companies
86+
companies, err := companyRepo.GetCompanies(ctx)
87+
if err != nil {
88+
log.WithFields(f).WithError(err).Warn("problem fetching companies")
89+
return
90+
}
91+
92+
invalidSignatures := make([]*signatures.ItemSignature, 0)
93+
94+
log.WithFields(f).Infof("processing %d companies", len(companies.Companies))
95+
// Fetch the ecla records
96+
97+
for _, company := range companies.Companies {
98+
// log.WithFields(f).Infof("processing company: %s", company.CompanyName)
99+
// Fetch the ecla records for the company
100+
invalidCompanyEcla := make([]*signatures.ItemSignature, 0)
101+
eclaRecords, err := signatureRepo.GetCompanyECLASignatures(ctx, company.CompanyID)
102+
if err != nil {
103+
log.WithFields(f).WithError(err).Warn("problem fetching ecla records")
104+
return
105+
}
106+
// log.WithFields(f).Infof("processing %d records for company: %s", len(eclaRecords), company.CompanyID)
107+
108+
if len(eclaRecords) == 0 {
109+
log.WithFields(f).Infof("no records for company: %s", company.CompanyID)
110+
continue
111+
}
112+
113+
var wg sync.WaitGroup
114+
var mu sync.Mutex
115+
for _, eclaRecord := range eclaRecords {
116+
wg.Add(1)
117+
go func(eclaRecord *signatures.ItemSignature) {
118+
defer wg.Done()
119+
// log.WithFields(f).Infof("processing signature record: %s", eclaRecord.SignatureID)
120+
if eclaRecord.DateCreated == "" {
121+
log.WithFields(f).Warnf("invalid signature record: %s", eclaRecord.SignatureID)
122+
mu.Lock()
123+
invalidCompanyEcla = append(invalidCompanyEcla, eclaRecord)
124+
mu.Unlock()
125+
return
126+
}
127+
}(eclaRecord)
128+
}
129+
wg.Wait()
130+
131+
invalidSignatures = append(invalidSignatures, invalidCompanyEcla...)
132+
133+
log.WithFields(f).Debugf("Found %d invalid eclas for company: %s", len(invalidCompanyEcla), company.CompanyID)
134+
}
135+
136+
log.WithFields(f).Infof("Processesing %d invalid records ", len(invalidSignatures))
137+
// For each ecla record, check the events table for the corresponding event and update the signature table
138+
update := 0
139+
140+
for _, eclaRecord := range invalidSignatures {
141+
events, err := getEvents(ctx, *eclaRecord)
142+
if err != nil {
143+
log.WithFields(f).WithError(err).Warnf("problem fetching events for ecla record: %s", eclaRecord.SignatureID)
144+
return
145+
}
146+
147+
if len(events) == 0 {
148+
log.WithFields(f).Warnf("no events found for ecla record: %s and user_id: %s", eclaRecord.SignatureID, eclaRecord.SignatureReferenceID)
149+
continue
150+
}
151+
152+
log.WithFields(f).Infof("found %d events for ecla record: %s", len(events), eclaRecord.SignatureID)
153+
154+
latestEvent := getLatestEvent(events)
155+
if latestEvent == nil {
156+
log.WithFields(f).Warnf("no latest event found for ecla record: %s", eclaRecord.SignatureID)
157+
continue
158+
}
159+
160+
log.Debugf("Found event:%s for user %s", latestEvent.EventID, eclaRecord.SignatureReferenceID)
161+
162+
// Update the signature record
163+
update++
164+
165+
update := map[string]interface{}{
166+
"date_created": latestEvent.EventTime,
167+
"date_modified": latestEvent.EventTime,
168+
}
169+
170+
log.WithFields(f).Infof("updating signature record: %s with : %+v", eclaRecord.SignatureID, update)
171+
172+
if !*dryRun {
173+
err = signatureRepo.UpdateSignature(ctx, eclaRecord.SignatureID, update)
174+
if err != nil {
175+
log.WithFields(f).WithError(err).Warnf("problem updating signature record: %s", eclaRecord.SignatureID)
176+
return
177+
}
178+
} else {
179+
log.WithFields(f).Info("dry run....")
180+
}
181+
182+
log.WithFields(f).Infof("updated signature record: %s", eclaRecord.SignatureID)
183+
}
184+
185+
log.WithFields(f).Infof("updated :%d records", update)
186+
}
187+
188+
func getEvents(ctx context.Context, ecla signatures.ItemSignature) ([]*models.Event, error) {
189+
190+
f := logrus.Fields{
191+
"functionName": "getEvents",
192+
"companyID": ecla.SignatureUserCompanyID,
193+
"projectID": ecla.SignatureProjectID,
194+
"userID": ecla.SignatureReferenceID,
195+
}
196+
197+
log.WithFields(f).Debug("searching events")
198+
199+
eventType := "EmployeeSignatureCreated"
200+
201+
claGroupModel, err := v1ProjectRepo.GetCLAGroupByID(ctx, ecla.SignatureProjectID, false)
202+
if err != nil {
203+
log.WithFields(f).Debugf("Unable to fetch cla group model")
204+
return nil, err
205+
}
206+
207+
eventParams := eventOps.SearchEventsParams{
208+
UserID: &ecla.SignatureReferenceID,
209+
ProjectSFID: &claGroupModel.ProjectExternalID,
210+
EventType: &eventType,
211+
CompanyID: &ecla.SignatureUserCompanyID,
212+
}
213+
214+
events, err := eventsService.SearchEvents(&eventParams)
215+
if err != nil {
216+
log.WithFields(f).WithError(err).Infof("Unable to get clagroup by id: %s", claGroupModel.ProjectID)
217+
return nil, err
218+
}
219+
220+
return events.Events, nil
221+
222+
}
223+
224+
func getLatestEvent(events []*models.Event) *models.Event {
225+
// Fetch the latest event from the list of events
226+
var latestEvent *models.Event
227+
for _, event := range events {
228+
if latestEvent == nil {
229+
latestEvent = event
230+
continue
231+
}
232+
if event.EventTime > latestEvent.EventTime {
233+
latestEvent = event
234+
}
235+
}
236+
return latestEvent
237+
}

‎cla-backend-go/signatures/mocks/mock_repo.go‎

Lines changed: 15 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎cla-backend-go/signatures/repository.go‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ type SignatureRepository interface {
106106
EclaAutoCreate(ctx context.Context, signatureID string, autoCreateECLA bool) error
107107
ActivateSignature(ctx context.Context, signatureID string) error
108108
GetICLAByDate(ctx context.Context, startDate string) ([]ItemSignature, error)
109+
GetCompanyECLASignatures(ctx context.Context, companyID string) ([]*ItemSignature, error)
109110
}
110111

111112
type iclaSignatureWithDetails struct {
@@ -173,6 +174,50 @@ func (repo repository) CreateSignature(ctx context.Context, signature *ItemSigna
173174

174175
}
175176

177+
func (repo repository) GetCompanyECLASignatures(ctx context.Context, companyID string) ([]*ItemSignature, error) {
178+
f := logrus.Fields{
179+
"functionName": "v1.signatures.repository.GetCompanyECLASignatures",
180+
"companyID": companyID,
181+
}
182+
log.WithFields(f).Debugf("fetching company ecla records")
183+
var itemSignatures []*ItemSignature
184+
185+
condition := expression.Key("signature_user_ccla_company_id").Equal(expression.Value(companyID))
186+
// filter := expression.Name("signature_signed").Equal(expression.Value(true)).And(expression.Name("signature_approved").Equal(expression.Value(true)))
187+
188+
expr, err := expression.NewBuilder().WithKeyCondition(condition).Build()
189+
if err != nil {
190+
return nil, err
191+
}
192+
193+
// Assemble the query input parameters
194+
queryInput := &dynamodb.QueryInput{
195+
ExpressionAttributeNames: expr.Names(),
196+
ExpressionAttributeValues: expr.Values(),
197+
KeyConditionExpression: expr.KeyCondition(),
198+
// FilterExpression: expr.Filter(),
199+
TableName: aws.String(repo.signatureTableName),
200+
IndexName: aws.String("signature-user-ccla-company-index"),
201+
}
202+
203+
results, err := repo.dynamoDBClient.Query(queryInput)
204+
if err != nil {
205+
return nil, err
206+
}
207+
208+
// No match, didn't find it
209+
if *results.Count == 0 {
210+
return nil, nil
211+
}
212+
213+
err = dynamodbattribute.UnmarshalListOfMaps(results.Items, &itemSignatures)
214+
if err != nil {
215+
return nil, err
216+
}
217+
218+
return itemSignatures, nil
219+
}
220+
176221
// GetItemSignature returns the signature for the specified signature id
177222
func (repo repository) GetItemSignature(ctx context.Context, signatureID string) (*ItemSignature, error) {
178223
f := logrus.Fields{

0 commit comments

Comments
 (0)