Certz2.1 updated the timeout and changed the error response as fatal

feature/gnsi/certz/tests/internal/setup_service/setup_service.go

@@ -59,7 +59,7 @@
 	*creds.UserPass
 }
 
 func (r *rpcCredentials) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) {
 	return map[string]string{
 		"username": r.UserPass.Username,
 		"password": r.UserPass.Password,
@@ -97,7 +97,7 @@
 	TrustBundleFile string
 }
 
 // CreateDialOptions function to create the gRPC dial options for certz client and retruns connection handle.
 func CreateNewDialOption(t *testing.T, newClientCert tls.Certificate, newCaCert *x509.CertPool, san, username, password, serverAddr string) (conn *grpc.ClientConn) {
 	credOpts := []grpc.DialOption{grpc.WithTransportCredentials(credentials.NewTLS(
 		&tls.Config{
@@ -212,7 +212,7 @@
 	return certzpb.CertificateChain{}
 }
 
 // LoadTrustBundle reads a file that contains a PKCS#7 trust‑bundle.
 func Loadpkcs7TrustBundle(path string) ([]*x509.Certificate, []byte, error) {
 	data, err := os.ReadFile(path)
 	if err != nil {
@@ -294,6 +294,11 @@
 		t.Logf("gNMI config is replaced with new ssl profile %s successfully.", profileID)
 		time.Sleep(30 * time.Second) //waiting 30s for gnmi config propagation//
 	}
+	if scale {
+		// waiting 180s for large scale certs to propagate after rotate response
+		t.Logf("STATUS:%s waiting 180s for large-scale 1000certs to propagate", time.Now())
+		time.Sleep(180 * time.Second)
+	}
 	//Verify gNSI service with new TLS credentials in loop with retries before finalize.
 	if success = VerifyGnsi(t, newcaCert, san, serverAddr, username, password, newclientCert, mismatch); !success {
 		t.Fatalf("gNSI service RPC  did not succeed after rotate. Certz/Rotate failed. FinalizeRequest will not be sent")
@@ -424,7 +429,7 @@
 }
 
 // ValidateGnsiAuthzGetRequest function to verify get request with authz client.
 func ValidateGnsiAuthzGetRequest(ctx context.Context, t *testing.T, authzClient authzpb.AuthzClient, mismatch bool) bool {
 
 	t.Logf("Verifying gNSI Authz GetRequest.")
 	rsp, err := authzClient.Get(ctx, &authzpb.GetRequest{})
@@ -462,7 +467,7 @@
 // Returns:
 //
 //	bool - True if verification succeeds or expected mismatch occurs, false otherwise.
 func VerifyGnoi(t *testing.T, caCert *x509.CertPool, san, serverAddr, username, password string, cert tls.Certificate, mismatch bool) bool {
 
 	credOpts := []grpc.DialOption{grpc.WithTransportCredentials(credentials.NewTLS(
 		&tls.Config{
@@ -489,7 +494,7 @@
 }
 
 // ValidateGnoiPingRequest function verifies ping request with the gnoiclient.
 func ValidateGnoiPingRequest(ctx context.Context, t *testing.T, sysClient spb.SystemClient, mismatch bool) bool {
 
 	t.Logf("Verifying gNOI Ping Request.")
 	if _, err := sysClient.Ping(ctx, &spb.PingRequest{}); err != nil {
@@ -546,12 +551,12 @@
 }
 
 // ValidateGnmiCapabilityRequest function validates the gNMI RPC request.
 func ValidateGnmiCapabilityRequest(ctx context.Context, t *testing.T, gnmiClient gnmipb.GNMIClient, mismatch bool) bool {
 
 	t.Logf("Verifying gNMI Capability Request.")
 	response, err := gnmiClient.Capabilities(ctx, &gnmipb.CapabilityRequest{})
 	if err != nil {
-		t.Errorf("gNMI Capability request failed with err: %v", err)
+		t.Fatalf("gNMI Capability request failed with err: %v", err)
 	}
 	t.Logf("VerifyGnmi:gNMI response: %s", response.GNMIVersion)
 	return true
@@ -605,7 +610,7 @@
 }
 
 // ValidateGribiGetRequest function verifies get request RPC with the gNMIClient.
 func ValidateGribiGetRequest(ctx context.Context, t *testing.T, gRibiClient gribipb.GRIBIClient, mismatch bool) bool {
 
 	t.Logf("Verifying gRIBI GetRequest.")
 	_, err := gRibiClient.Get(ctx, &gribipb.GetRequest{})
@@ -660,11 +665,12 @@
 }
 
 // ValidateP4RtCapabilitiesRequest function verifies the Capabilities request with the p4RT client.
 func ValidateP4RtCapabilitiesRequest(ctx context.Context, t *testing.T, p4rtClient p4rtpb.P4RuntimeClient, mismatch bool) bool {
 
 	t.Logf("Verifying P4Rt Capability Request.")
+
 	if _, err := p4rtClient.Capabilities(ctx, &p4rtpb.CapabilitiesRequest{}); err != nil {
-		t.Errorf("Failed to connect P4rtClient with error %v.", err)
+		t.Fatalf("Failed to connect P4rtClient with error %v.", err)
 	}
 	return true
 }

feature/gnsi/certz/tests/server_certificates/server_certificates_test.go

@@ -32,7 +32,7 @@ import (
 
 const (
 	dirPath                  = "../../test_data/"
-	timeOutVar time.Duration = 2 * time.Minute
+	timeOutVar time.Duration = 5 * time.Minute
 )
 
 // DUTCredentialer is an interface for getting credentials from a DUT binding.
@@ -48,7 +48,6 @@ var (
 	prevClientCertFile  string          = ""
 	prevClientKeyFile   string          = ""
 	prevTrustBundleFile string          = ""
-	logTime             string          = time.Now().String() //Timestamp
 	expectedResult      bool            = true
 )
 
@@ -64,37 +63,37 @@ func TestServerCert(t *testing.T) {
 	dut := ondatra.DUT(t, "dut")
 	serverAddr = dut.Name() //returns the device name.
 	if err := binding.DUTAs(dut.RawAPIs().BindingDUT(), &creds); err != nil {
-		t.Fatalf("%s:STATUS:Failed to get DUT credentials using binding.DUTAs: %v. The binding for %s must implement the DUTCredentialer interface", logTime, err, dut.Name())
+		t.Fatalf("%s:STATUS:Failed to get DUT credentials using binding.DUTAs: %v. The binding for %s must implement the DUTCredentialer interface", time.Now().String(), err, dut.Name())
 	}
 	username := creds.RPCUsername()
 	password := creds.RPCPassword()
 	t.Logf("Validation of all services that are using gRPC before server certificate rotation.")
 	gnmiClient, gnsiC := setupService.PreInitCheck(context.Background(), t, dut)
 	//Generate testdata certificates.
-	t.Logf("%s:STATUS:Generation of test data certificates.", logTime)
+	t.Logf("%s:STATUS:Generation of test data certificates.", time.Now().String())
 	if err := setupService.TestdataMakeCleanup(t, dirPath, timeOutVar, "./mk_cas.sh"); err != nil {
-		t.Logf("%s:STATUS:Generation of testdata certificates failed!: %v", logTime, err)
+		t.Logf("%s:STATUS:Generation of testdata certificates failed!: %v", time.Now().String(), err)
 	}
 	//Create a certz client.
 	ctx := context.Background()
 	certzClient := gnsiC.Certz()
-	t.Logf("%s:STATUS:Precheck:checking baseline sslprofile list.", logTime)
+	t.Logf("%s:STATUS:Precheck:checking baseline sslprofile list.", time.Now().String())
 	//Get sslprofile list.
 	if getResp := setupService.GetSslProfilelist(ctx, t, certzClient, &certzpb.GetProfileListRequest{}); slices.Contains(getResp.SslProfileIds, testProfile) {
-		t.Fatalf("%s:STATUS:profileID %s already exists.", logTime, testProfile)
+		t.Fatalf("%s:STATUS:profileID %s already exists.", time.Now().String(), testProfile)
 	}
 	//Add a new sslprofileID.
-	t.Logf("%s:STATUS:Adding new sslprofileID %s.", logTime, testProfile)
+	t.Logf("%s:STATUS:Adding new sslprofileID %s.", time.Now().String(), testProfile)
 	if addProfileResponse, err := certzClient.AddProfile(ctx, &certzpb.AddProfileRequest{SslProfileId: testProfile}); err != nil {
-		t.Fatalf("%s:STATUS:Add profile request failed with %v! ", logTime, err)
+		t.Fatalf("%s:STATUS:Add profile request failed with %v! ", time.Now().String(), err)
 	} else {
-		t.Logf("%s:STATUS:Received the AddProfileResponse %v.", logTime, addProfileResponse)
+		t.Logf("%s:STATUS:Received the AddProfileResponse %v.", time.Now().String(), addProfileResponse)
 	}
 	//Get sslprofile list after new sslprofile addition.
 	if getResp := setupService.GetSslProfilelist(ctx, t, certzClient, &certzpb.GetProfileListRequest{}); !slices.Contains(getResp.SslProfileIds, testProfile) {
-		t.Fatalf("%s:STATUS:newly added profileID is not seen.", logTime)
+		t.Fatalf("%s:STATUS:newly added profileID is not seen.", time.Now().String())
 	} else {
-		t.Logf("%sSTATUS:new profileID %s is seen in sslprofile list", logTime, testProfile)
+		t.Logf("%sSTATUS:new profileID %s is seen in sslprofile list", time.Now().String(), testProfile)
 	}
 	cases := []struct {
 		desc            string
@@ -225,7 +224,7 @@ func TestServerCert(t *testing.T) {
 	}
 	for _, tc := range cases {
 		t.Run(tc.desc, func(t *testing.T) {
-			t.Logf("%s:STATUS:Starting test case: %s", logTime, tc.desc)
+			t.Logf("%s:STATUS:Starting test case: %s", time.Now().String(), tc.desc)
 			//Read the serverSAN (Subject Alternative Name) from the certificate used for TLS verification.
 			serverSAN := setupService.ReadDecodeServerCertificate(t, tc.serverCertFile)
 			//Build serverCertEntity for the server certificate rotation.
@@ -237,7 +236,7 @@ func TestServerCert(t *testing.T) {
 			//Create a new Cert Pool and add the certs from the trust bundle.
 			pkcs7certs, pkcs7data, err := setupService.Loadpkcs7TrustBundle(tc.trustBundleFile)
 			if err != nil {
-				t.Fatalf("%s:STATUS:Failed to load trust bundle: %v", logTime, err)
+				t.Fatalf("%s:STATUS:Failed to load trust bundle: %v", time.Now().String(), err)
 			}
 			newCaCert := x509.NewCertPool()
 			for _, c := range pkcs7certs {
@@ -248,18 +247,18 @@ func TestServerCert(t *testing.T) {
 			//Load Client certificate.
 			newClientCert, err := tls.LoadX509KeyPair(tc.clientCertFile, tc.clientKeyFile)
 			if err != nil {
-				t.Fatalf("%s:STATUS:Failed to load client cert: %v", logTime, err)
+				t.Fatalf("%s:STATUS:Failed to load client cert: %v", time.Now().String(), err)
 			}
 			if tc.newTLScreds {
-				t.Logf("%s:STATUS:%sCreating new TLS credentials for client connection.", logTime, tc.desc)
+				t.Logf("%s:STATUS:%sCreating new TLS credentials for client connection.", time.Now().String(), tc.desc)
 				//Load the prior client keypair for new client TLS credentials.
 				prevClientCert, err := tls.LoadX509KeyPair(prevClientCertFile, prevClientKeyFile)
 				if err != nil {
-					t.Fatalf("%s:STATUS:%s:Failed to load previous client cert: %v.", logTime, tc.desc, err)
+					t.Fatalf("%s:STATUS:%s:Failed to load previous client cert: %v.", time.Now().String(), tc.desc, err)
 				}
 				oldPkcs7certs, oldPkcs7data, err := setupService.Loadpkcs7TrustBundle(prevTrustBundleFile)
 				if err != nil {
-					t.Fatalf("%s:STATUS:%s:Failed to load previous trust bundle,data %v with %v.", logTime, tc.desc, oldPkcs7data, err)
+					t.Fatalf("%s:STATUS:%s:Failed to load previous trust bundle,data %v with %v.", time.Now().String(), tc.desc, oldPkcs7data, err)
 				}
 				//Create a old set of Cert Pool and append the certs from previous trust bundle.
 				prevCaCert := x509.NewCertPool()
@@ -268,7 +267,7 @@ func TestServerCert(t *testing.T) {
 				}
 				//Before rotation, validation of all services with existing certificates.
 				if result := setupService.ServicesValidationCheck(t, prevCaCert, expectedResult, serverSAN, serverAddr, username, password, prevClientCert, tc.mismatch); !result {
-					t.Fatalf("%s:STATUS:%s:service validation failed before rotate- got %v, want %v.", logTime, tc.desc, result, expectedResult)
+					t.Fatalf("%s:STATUS:%s:service validation failed before rotate- got %v, want %v.", time.Now().String(), tc.desc, result, expectedResult)
 				}
 				//Retrieve the connection with previous TLS credentials for certz rotation.
 				conn := setupService.CreateNewDialOption(t, prevClientCert, prevCaCert, serverSAN, username, password, serverAddr)
@@ -277,31 +276,31 @@ func TestServerCert(t *testing.T) {
 				certzClient = certzpb.NewCertzClient(conn)
 				gnmiClient = gnmi.NewGNMIClient(conn)
 			} else {
-				t.Logf("%s:STATUS:%s:Using existing TLS credentials for client connection in first iteration.", logTime, tc.desc)
+				t.Logf("%s:STATUS:%s:Using existing TLS credentials for client connection in first iteration.", time.Now().String(), tc.desc)
 			}
 			//Initiate server certificate rotation.
-			t.Logf("%s:STATUS:%s Initiating Certz rotation with server cert: %s and trust bundle: %s.", logTime, tc.desc, tc.serverCertFile, tc.trustBundleFile)
+			t.Logf("%s:STATUS:%s Initiating Certz rotation with server cert: %s and trust bundle: %s.", time.Now().String(), tc.desc, tc.serverCertFile, tc.trustBundleFile)
 			if success := setupService.CertzRotate(ctx, t, newCaCert, certzClient, gnmiClient, newClientCert, dut, username, password, serverSAN, serverAddr, testProfile, tc.newTLScreds, tc.mismatch, tc.scale, &serverCertEntity, &trustBundleEntity); !success {
-				t.Fatalf("%s:STATUS: %s:Certz rotation failed.", logTime, tc.desc)
+				t.Fatalf("%s:STATUS: %s:Certz rotation failed.", time.Now().String(), tc.desc)
 			}
-			t.Logf("%s:STATUS:%s: Certz rotation completed!", logTime, tc.desc)
+			t.Logf("%s:STATUS:%s: Certz rotation completed!", time.Now().String(), tc.desc)
 			//Post rotate validation of all services.
 			t.Run("Verification of new connection after rotate", func(t *testing.T) {
 				if result := setupService.ServicesValidationCheck(t, newCaCert, expectedResult, serverSAN, serverAddr, username, password, newClientCert, tc.mismatch); !result {
 					t.Fatalf("STATUS:%s:service validation failed after rotate- got %v, want %v.", tc.desc, result, expectedResult)
 				}
-				t.Logf("%s:STATUS:%s:service validation done!", logTime, tc.desc)
+				t.Logf("%s:STATUS:%s:service validation done!", time.Now().String(), tc.desc)
 			})
 			//Archiving previous client cert/key and trustbundle.
 			prevClientCertFile = tc.clientCertFile
 			prevClientKeyFile = tc.clientKeyFile
 			prevTrustBundleFile = tc.trustBundleFile
 		})
 	}
-	t.Logf("%s:STATUS:Cleanup of test data.", logTime)
+	t.Logf("%s:STATUS:Cleanup of test data.", time.Now().String())
 	//Cleanup of test data.
 	if err := setupService.TestdataMakeCleanup(t, dirPath, timeOutVar, "./cleanup.sh"); err != nil {
-		t.Logf("%s:STATUS:Cleanup of testdata certificates failed!: %v", logTime, err)
+		t.Logf("%s:STATUS:Cleanup of testdata certificates failed!: %v", time.Now().String(), err)
 	}
-	t.Logf("%s:STATUS:Test completed!", logTime)
+	t.Logf("%s:STATUS:Test completed!", time.Now().String())
 }