Process committees correctly post-electra

Author: jshufro

shared/services/beacon/client.go

@@ -82,6 +82,9 @@ type Committees interface {
 	// Validators returns the list of validators of the committee at
 	// the provided offset
 	Validators(int) []string
+	// ValidatorCount returns the number of validators in the committee at
+	// the provided offset
+	ValidatorCount(int) int
 	// Count returns the number of committees in the response
 	Count() int
 	// Release returns the reused validators slice buffer to the pool for
@@ -94,6 +97,26 @@ type AttestationInfo struct {
 	AggregationBits bitfield.Bitlist
 	SlotIndex       uint64
 	CommitteeIndex  uint64
+	CommitteeBits   bitfield.Bitlist
+}
+
+// if !attestation.ValidatorAttested(i, position, slotInfo.CommitteeSizes) {
+func (a AttestationInfo) ValidatorAttested(committeeIndex int, position int, committeeSizes map[uint64]int) bool {
+	// Calculate the offset in aggregation_bits
+	var offset int
+	if a.CommitteeBits == nil {
+		offset = position
+	} else {
+		committeeOffset := 0
+		if committeeIndex > 0 {
+			for i := 0; i < committeeIndex; i++ {
+				committeeOffset += committeeSizes[uint64(i)]
+			}
+		}
+		offset = committeeOffset + position
+	}
+
+	return a.AggregationBits.BitAt(uint64(offset))
 }
 
 // Beacon client type

shared/services/beacon/client/committee.go

@@ -65,6 +65,10 @@ func (c *CommitteesResponse) Validators(idx int) []string {
 	return c.Data[idx].Validators
 }
 
+func (c *CommitteesResponse) ValidatorCount(idx int) int {
+	return len(c.Data[idx].Validators)
+}
+
 func (c *CommitteesResponse) Release() {
 	for _, committee := range c.Data {
 		// Reset the slice length to 0 (capacity stays the same)

shared/services/beacon/client/std-http-client.go

@@ -621,6 +621,12 @@ func (c *StandardHttpClient) GetAttestations(blockId string) ([]beacon.Attestati
 		if err != nil {
 			return nil, false, fmt.Errorf("Error decoding aggregation bits for attestation %d of block %s: %w", i, blockId, err)
 		}
+		if attestation.CommitteeBits != "" && attestation.CommitteeBits != "0x" {
+			attestationInfo[i].CommitteeBits, err = hex.DecodeString(attestation.CommitteeBits)
+			if err != nil {
+				return nil, false, fmt.Errorf("Error decoding committee bits for attestation %d of block %s: %w", i, blockId, err)
+			}
+		}
 	}
 
 	return attestationInfo, true, nil
@@ -661,6 +667,12 @@ func (c *StandardHttpClient) GetBeaconBlock(blockId string) (beacon.BeaconBlock,
 		if err != nil {
 			return beacon.BeaconBlock{}, false, fmt.Errorf("Error decoding aggregation bits for attestation %d of block %s: %w", i, blockId, err)
 		}
+		if attestation.CommitteeBits != "" && attestation.CommitteeBits != "0x" {
+			info.CommitteeBits, err = hex.DecodeString(attestation.CommitteeBits)
+			if err != nil {
+				return beacon.BeaconBlock{}, false, fmt.Errorf("Error decoding committee bits for attestation %d of block %s: %w", i, blockId, err)
+			}
+		}
 		beaconBlock.Attestations = append(beaconBlock.Attestations, info)
 	}
 

shared/services/beacon/client/types.go

@@ -166,6 +166,7 @@ type Attestation struct {
 		Slot  uinteger `json:"slot"`
 		Index uinteger `json:"index"`
 	} `json:"data"`
+	CommitteeBits string `json:"committee_bits"`
 }
 
 type Withdrawal struct {

shared/services/rewards/generator-impl-v8.go

@@ -852,50 +852,58 @@ func (r *treeGeneratorImpl_v8) checkDutiesForSlot(attestations []beacon.Attestat
 		if inclusionSlot-attestation.SlotIndex > r.beaconConfig.SlotsPerEpoch {
 			continue
 		}
-		rpCommittee, exists := slotInfo.Committees[attestation.CommitteeIndex]
-		if !exists {
-			continue
+		var committeeIndexes []int
+		if attestation.CommitteeBits == nil {
+			committeeIndexes = []int{int(attestation.CommitteeIndex)}
+		} else {
+			committeeIndexes = attestation.CommitteeBits.BitIndices()
 		}
-		blockTime := r.genesisTime.Add(time.Second * time.Duration(r.networkState.BeaconConfig.SecondsPerSlot*attestation.SlotIndex))
-
-		// Check if each RP validator attested successfully
-		for position, validator := range rpCommittee.Positions {
-			if !attestation.AggregationBits.BitAt(uint64(position)) {
+		for _, committeeIndex := range committeeIndexes {
+			rpCommittee, exists := slotInfo.Committees[uint64(committeeIndex)]
+			if !exists {
 				continue
 			}
+			blockTime := r.genesisTime.Add(time.Second * time.Duration(r.networkState.BeaconConfig.SecondsPerSlot*attestation.SlotIndex))
 
-			// This was seen, so remove it from the missing attestations and add it to the completed ones
-			delete(rpCommittee.Positions, position)
-			if len(rpCommittee.Positions) == 0 {
-				delete(slotInfo.Committees, attestation.CommitteeIndex)
-			}
-			if len(slotInfo.Committees) == 0 {
-				delete(r.intervalDutiesInfo.Slots, attestation.SlotIndex)
-			}
-			delete(validator.MissingAttestationSlots, attestation.SlotIndex)
+			// Check if each RP validator attested successfully
+			for position, validator := range rpCommittee.Positions {
+				if !attestation.ValidatorAttested(committeeIndex, position, slotInfo.CommitteeSizes) {
+					continue
+				}
 
-			// Check if this minipool was opted into the SP for this block
-			nodeDetails := r.nodeDetails[validator.NodeIndex]
-			if blockTime.Sub(nodeDetails.OptInTime) < 0 || nodeDetails.OptOutTime.Sub(blockTime) < 0 {
-				// Not opted in
-				continue
-			}
+				// This was seen, so remove it from the missing attestations and add it to the completed ones
+				delete(rpCommittee.Positions, position)
+				if len(rpCommittee.Positions) == 0 {
+					delete(slotInfo.Committees, uint64(committeeIndex))
+				}
+				if len(slotInfo.Committees) == 0 {
+					delete(r.intervalDutiesInfo.Slots, attestation.SlotIndex)
+				}
+				delete(validator.MissingAttestationSlots, attestation.SlotIndex)
+
+				// Check if this minipool was opted into the SP for this block
+				nodeDetails := r.nodeDetails[validator.NodeIndex]
+				if blockTime.Sub(nodeDetails.OptInTime) < 0 || nodeDetails.OptOutTime.Sub(blockTime) < 0 {
+					// Not opted in
+					continue
+				}
 
-			// Mark this duty as completed
-			validator.CompletedAttestations[attestation.SlotIndex] = true
-
-			// Get the pseudoscore for this attestation
-			details := r.networkState.MinipoolDetailsByAddress[validator.Address]
-			bond, fee := r.getMinipoolBondAndNodeFee(details, blockTime)
-			minipoolScore := big.NewInt(0).Sub(one, fee)   // 1 - fee
-			minipoolScore.Mul(minipoolScore, bond)         // Multiply by bond
-			minipoolScore.Div(minipoolScore, validatorReq) // Divide by 32 to get the bond as a fraction of a total validator
-			minipoolScore.Add(minipoolScore, fee)          // Total = fee + (bond/32)(1 - fee)
-
-			// Add it to the minipool's score and the total score
-			validator.AttestationScore.Add(&validator.AttestationScore.Int, minipoolScore)
-			r.totalAttestationScore.Add(r.totalAttestationScore, minipoolScore)
-			r.successfulAttestations++
+				// Mark this duty as completed
+				validator.CompletedAttestations[attestation.SlotIndex] = true
+
+				// Get the pseudoscore for this attestation
+				details := r.networkState.MinipoolDetailsByAddress[validator.Address]
+				bond, fee := r.getMinipoolBondAndNodeFee(details, blockTime)
+				minipoolScore := big.NewInt(0).Sub(one, fee)   // 1 - fee
+				minipoolScore.Mul(minipoolScore, bond)         // Multiply by bond
+				minipoolScore.Div(minipoolScore, validatorReq) // Divide by 32 to get the bond as a fraction of a total validator
+				minipoolScore.Add(minipoolScore, fee)          // Total = fee + (bond/32)(1 - fee)
+
+				// Add it to the minipool's score and the total score
+				validator.AttestationScore.Add(&validator.AttestationScore.Int, minipoolScore)
+				r.totalAttestationScore.Add(r.totalAttestationScore, minipoolScore)
+				r.successfulAttestations++
+			}
 		}
 	}
 
@@ -916,6 +924,18 @@ func (r *treeGeneratorImpl_v8) getDutiesForEpoch(committees beacon.Committees) e
 		blockTime := r.genesisTime.Add(time.Second * time.Duration(r.beaconConfig.SecondsPerSlot*slotIndex))
 		committeeIndex := committees.Index(idx)
 
+		// Add the committee size to the list, for calculating offset in post-electra aggregation_bits
+		slotInfo, exists := r.intervalDutiesInfo.Slots[slotIndex]
+		if !exists {
+			slotInfo = &SlotInfo{
+				Index:          slotIndex,
+				Committees:     map[uint64]*CommitteeInfo{},
+				CommitteeSizes: map[uint64]int{},
+			}
+			r.intervalDutiesInfo.Slots[slotIndex] = slotInfo
+		}
+		slotInfo.CommitteeSizes[committeeIndex] = committees.ValidatorCount(idx)
+
 		// Check if there are any RP validators in this committee
 		rpValidators := map[int]*MinipoolInfo{}
 		for position, validator := range committees.Validators(idx) {
@@ -948,14 +968,6 @@ func (r *treeGeneratorImpl_v8) getDutiesForEpoch(committees beacon.Committees) e
 
 		// If there are some RP validators, add this committee to the map
 		if len(rpValidators) > 0 {
-			slotInfo, exists := r.intervalDutiesInfo.Slots[slotIndex]
-			if !exists {
-				slotInfo = &SlotInfo{
-					Index:      slotIndex,
-					Committees: map[uint64]*CommitteeInfo{},
-				}
-				r.intervalDutiesInfo.Slots[slotIndex] = slotInfo
-			}
 			slotInfo.Committees[committeeIndex] = &CommitteeInfo{
 				Index:     committeeIndex,
 				Positions: rpValidators,

shared/services/rewards/generator-impl-v9-v10.go

@@ -948,58 +948,67 @@ func (r *treeGeneratorImpl_v9_v10) checkAttestations(attestations []beacon.Attes
 		if inclusionSlot-attestation.SlotIndex > r.beaconConfig.SlotsPerEpoch {
 			continue
 		}
-		rpCommittee, exists := slotInfo.Committees[attestation.CommitteeIndex]
-		if !exists {
-			continue
+		var committeeIndexes []int
+		if attestation.CommitteeBits == nil {
+			committeeIndexes = []int{int(attestation.CommitteeIndex)}
+		} else {
+			committeeIndexes = attestation.CommitteeBits.BitIndices()
 		}
-		blockTime := r.genesisTime.Add(time.Second * time.Duration(r.networkState.BeaconConfig.SecondsPerSlot*attestation.SlotIndex))
 
-		// Check if each RP validator attested successfully
-		for position, validator := range rpCommittee.Positions {
-			if !attestation.AggregationBits.BitAt(uint64(position)) {
+		for _, committeeIndex := range committeeIndexes {
+			rpCommittee, exists := slotInfo.Committees[uint64(committeeIndex)]
+			if !exists {
 				continue
 			}
+			blockTime := r.genesisTime.Add(time.Second * time.Duration(r.networkState.BeaconConfig.SecondsPerSlot*attestation.SlotIndex))
 
-			// This was seen, so remove it from the missing attestations and add it to the completed ones
-			delete(rpCommittee.Positions, position)
-			if len(rpCommittee.Positions) == 0 {
-				delete(slotInfo.Committees, attestation.CommitteeIndex)
-			}
-			if len(slotInfo.Committees) == 0 {
-				delete(r.intervalDutiesInfo.Slots, attestation.SlotIndex)
-			}
-			delete(validator.MissingAttestationSlots, attestation.SlotIndex)
+			// Check if each RP validator attested successfully
+			for position, validator := range rpCommittee.Positions {
+				if !attestation.ValidatorAttested(committeeIndex, position, slotInfo.CommitteeSizes) {
+					continue
+				}
 
-			// Check if this minipool was opted into the SP for this block
-			nodeDetails := r.nodeDetails[validator.NodeIndex]
-			if blockTime.Before(nodeDetails.OptInTime) || blockTime.After(nodeDetails.OptOutTime) {
-				// Not opted in
-				continue
-			}
+				// This was seen, so remove it from the missing attestations and add it to the completed ones
+				delete(rpCommittee.Positions, position)
+				if len(rpCommittee.Positions) == 0 {
+					delete(slotInfo.Committees, uint64(committeeIndex))
+				}
+				if len(slotInfo.Committees) == 0 {
+					delete(r.intervalDutiesInfo.Slots, attestation.SlotIndex)
+				}
+				delete(validator.MissingAttestationSlots, attestation.SlotIndex)
 
-			eligibleBorrowedEth := nodeDetails.EligibleBorrowedEth
-			_, percentOfBorrowedEth := r.networkState.GetStakedRplValueInEthAndPercentOfBorrowedEth(eligibleBorrowedEth, nodeDetails.RplStake)
+				// Check if this minipool was opted into the SP for this block
+				nodeDetails := r.nodeDetails[validator.NodeIndex]
+				if blockTime.Before(nodeDetails.OptInTime) || blockTime.After(nodeDetails.OptOutTime) {
+					// Not opted in
+					continue
+				}
 
-			// Mark this duty as completed
-			validator.CompletedAttestations[attestation.SlotIndex] = true
+				eligibleBorrowedEth := nodeDetails.EligibleBorrowedEth
+				_, percentOfBorrowedEth := r.networkState.GetStakedRplValueInEthAndPercentOfBorrowedEth(eligibleBorrowedEth, nodeDetails.RplStake)
 
-			// Get the pseudoscore for this attestation
-			details := r.networkState.MinipoolDetailsByAddress[validator.Address]
-			bond, fee := details.GetMinipoolBondAndNodeFee(blockTime)
+				// Mark this duty as completed
+				validator.CompletedAttestations[attestation.SlotIndex] = true
 
-			if r.rewardsFile.RulesetVersion >= 10 {
-				fee = fees.GetMinipoolFeeWithBonus(bond, fee, percentOfBorrowedEth)
-			}
+				// Get the pseudoscore for this attestation
+				details := r.networkState.MinipoolDetailsByAddress[validator.Address]
+				bond, fee := details.GetMinipoolBondAndNodeFee(blockTime)
 
-			minipoolScore := big.NewInt(0).Sub(oneEth, fee) // 1 - fee
-			minipoolScore.Mul(minipoolScore, bond)          // Multiply by bond
-			minipoolScore.Div(minipoolScore, thirtyTwoEth)  // Divide by 32 to get the bond as a fraction of a total validator
-			minipoolScore.Add(minipoolScore, fee)           // Total = fee + (bond/32)(1 - fee)
+				if r.rewardsFile.RulesetVersion >= 10 {
+					fee = fees.GetMinipoolFeeWithBonus(bond, fee, percentOfBorrowedEth)
+				}
 
-			// Add it to the minipool's score and the total score
-			validator.AttestationScore.Add(&validator.AttestationScore.Int, minipoolScore)
-			r.totalAttestationScore.Add(r.totalAttestationScore, minipoolScore)
-			r.successfulAttestations++
+				minipoolScore := big.NewInt(0).Sub(oneEth, fee) // 1 - fee
+				minipoolScore.Mul(minipoolScore, bond)          // Multiply by bond
+				minipoolScore.Div(minipoolScore, thirtyTwoEth)  // Divide by 32 to get the bond as a fraction of a total validator
+				minipoolScore.Add(minipoolScore, fee)           // Total = fee + (bond/32)(1 - fee)
+
+				// Add it to the minipool's score and the total score
+				validator.AttestationScore.Add(&validator.AttestationScore.Int, minipoolScore)
+				r.totalAttestationScore.Add(r.totalAttestationScore, minipoolScore)
+				r.successfulAttestations++
+			}
 		}
 	}
 
@@ -1020,6 +1029,18 @@ func (r *treeGeneratorImpl_v9_v10) getDutiesForEpoch(committees beacon.Committee
 		blockTime := r.genesisTime.Add(time.Second * time.Duration(r.beaconConfig.SecondsPerSlot*slotIndex))
 		committeeIndex := committees.Index(idx)
 
+		// Add the committee size to the list, for calculating offset in post-electra aggregation_bits
+		slotInfo, exists := r.intervalDutiesInfo.Slots[slotIndex]
+		if !exists {
+			slotInfo = &SlotInfo{
+				Index:          slotIndex,
+				Committees:     map[uint64]*CommitteeInfo{},
+				CommitteeSizes: map[uint64]int{},
+			}
+			r.intervalDutiesInfo.Slots[slotIndex] = slotInfo
+		}
+		slotInfo.CommitteeSizes[committeeIndex] = committees.ValidatorCount(idx)
+
 		// Check if there are any RP validators in this committee
 		rpValidators := map[int]*MinipoolInfo{}
 		for position, validator := range committees.Validators(idx) {
@@ -1052,14 +1073,6 @@ func (r *treeGeneratorImpl_v9_v10) getDutiesForEpoch(committees beacon.Committee
 
 		// If there are some RP validators, add this committee to the map
 		if len(rpValidators) > 0 {
-			slotInfo, exists := r.intervalDutiesInfo.Slots[slotIndex]
-			if !exists {
-				slotInfo = &SlotInfo{
-					Index:      slotIndex,
-					Committees: map[uint64]*CommitteeInfo{},
-				}
-				r.intervalDutiesInfo.Slots[slotIndex] = slotInfo
-			}
 			slotInfo.Committees[committeeIndex] = &CommitteeInfo{
 				Index:     committeeIndex,
 				Positions: rpValidators,

shared/services/rewards/test/beacon.go

@@ -387,6 +387,10 @@ func (mbc *MockBeaconCommittees) Validators(index int) []string {
 	return mbc.slots[index].validators
 }
 
+func (mbc *MockBeaconCommittees) ValidatorCount(index int) int {
+	return len(mbc.slots[index].validators)
+}
+
 // Release is a no-op
 func (mbc *MockBeaconCommittees) Release() {
 }

shared/services/rewards/types.go

@@ -234,8 +234,9 @@ type IntervalDutiesInfo struct {
 }
 
 type SlotInfo struct {
-	Index      uint64
-	Committees map[uint64]*CommitteeInfo
+	Index          uint64
+	Committees     map[uint64]*CommitteeInfo
+	CommitteeSizes map[uint64]int
 }
 
 type CommitteeInfo struct {