diff --git a/feature/gnsi/certz/tests/internal/setup_service/setup_service.go b/feature/gnsi/certz/tests/internal/setup_service/setup_service.go index 9363ecec4c8..e8f35162f12 100644 --- a/feature/gnsi/certz/tests/internal/setup_service/setup_service.go +++ b/feature/gnsi/certz/tests/internal/setup_service/setup_service.go @@ -294,6 +294,11 @@ func CertzRotate(ctx context.Context, t *testing.T, newcaCert *x509.CertPool, ce t.Logf("gNMI config is replaced with new ssl profile %s successfully.", profileID) time.Sleep(30 * time.Second) //waiting 30s for gnmi config propagation// } + if scale { + // waiting 180s for large scale certs to propagate after rotate response + t.Logf("STATUS:%s waiting 180s for large-scale 1000certs to propagate", time.Now()) + time.Sleep(180 * time.Second) + } //Verify gNSI service with new TLS credentials in loop with retries before finalize. if success = VerifyGnsi(t, newcaCert, san, serverAddr, username, password, newclientCert, mismatch); !success { t.Fatalf("gNSI service RPC did not succeed after rotate. Certz/Rotate failed. FinalizeRequest will not be sent") diff --git a/feature/gnsi/certz/tests/trust_bundle/trustbundle_test.go b/feature/gnsi/certz/tests/trust_bundle/trustbundle_test.go index b5588158ae1..15dca2553bb 100644 --- a/feature/gnsi/certz/tests/trust_bundle/trustbundle_test.go +++ b/feature/gnsi/certz/tests/trust_bundle/trustbundle_test.go @@ -32,7 +32,7 @@ import ( const ( dirPath = "../../test_data/" - timeOutVar time.Duration = 2 * time.Minute + timeOutVar time.Duration = 180 * time.Minute ) // DUTCredentialer is an interface for getting credentials from a DUT binding. @@ -48,7 +48,6 @@ var ( prevClientCertFile string = "" prevClientKeyFile string = "" prevTrustBundleFile string = "" - logTime string = time.Now().String() //Timestamp expectedResult bool = true ) @@ -63,37 +62,37 @@ func TestTrustBundleCert(t *testing.T) { dut := ondatra.DUT(t, "dut") serverAddr = dut.Name() //returns the device name. if err := binding.DUTAs(dut.RawAPIs().BindingDUT(), &creds); err != nil { - t.Fatalf("%s:STATUS:Failed to get DUT credentials using binding.DUTAs: %v. The binding for %s must implement the DUTCredentialer interface.", logTime, err, dut.Name()) + t.Fatalf("STATUS:Failed to get DUT credentials using binding.DUTAs: %v. The binding for %s must implement the DUTCredentialer interface.", err, dut.Name()) } username := creds.RPCUsername() password := creds.RPCPassword() - t.Logf("%s:STATUS:Validation of all services that are using gRPC before certz rotation.", logTime) + t.Logf("%s:STATUS:Validation of all services that are using gRPC before certz rotation.", time.Now().String()) gnmiClient, gnsiC := setup_service.PreInitCheck(context.Background(), t, dut) //Generate testdata certificates. - t.Logf("%s:Creation of test data.", logTime) + t.Logf("%s:Creation of test data.", time.Now().String()) if err := setup_service.TestdataMakeCleanup(t, dirPath, timeOutVar, "./mk_cas.sh"); err != nil { - t.Logf("%s:STATUS:Generation of testdata certificates failed!: %v", logTime, err) + t.Logf("%s:STATUS:Generation of testdata certificates failed!: %v", time.Now().String(), err) } //Create a certz client. ctx := context.Background() certzClient := gnsiC.Certz() - t.Logf("%s:STATUS:Precheck:checking baseline sslprofile list.", logTime) + t.Logf("%s:STATUS:Precheck:checking baseline sslprofile list.", time.Now().String()) //Get sslprofile list. if getResp := setup_service.GetSslProfilelist(ctx, t, certzClient, &certzpb.GetProfileListRequest{}); slices.Contains(getResp.SslProfileIds, testProfile) { - t.Fatalf("%s:STATUS:profileID %s already exists.", logTime, testProfile) + t.Fatalf("%s:STATUS:profileID %s already exists.", time.Now().String(), testProfile) } //Add new sslprofileID. - t.Logf("%s:Adding new empty sslprofile ID %s.", logTime, testProfile) + t.Logf("%s:Adding new empty sslprofile ID %s.", time.Now().String(), testProfile) if addProfileResponse, err := certzClient.AddProfile(ctx, &certzpb.AddProfileRequest{SslProfileId: testProfile}); err != nil { - t.Fatalf("%s:STATUS:Add profile request failed with %v! ", logTime, err) + t.Fatalf("%s:STATUS:Add profile request failed with %v! ", time.Now().String(), err) } else { - t.Logf("%s:STATUS:Received the AddProfileResponse %v.", logTime, addProfileResponse) + t.Logf("%s:STATUS:Received the AddProfileResponse %v.", time.Now().String(), addProfileResponse) } //Get sslprofile list after new sslprofile addition. if getResp := setup_service.GetSslProfilelist(ctx, t, certzClient, &certzpb.GetProfileListRequest{}); !slices.Contains(getResp.SslProfileIds, testProfile) { - t.Fatalf("%s:STATUS:newly added profileID is not seen.", logTime) + t.Fatalf("%s:STATUS:newly added profileID is not seen.", time.Now().String()) } else { - t.Logf("%s:STATUS:new profileID %s is seen in sslprofile list", logTime, testProfile) + t.Logf("%s:STATUS:new profileID %s is seen in sslprofile list", time.Now().String(), testProfile) } cases := []struct { desc string @@ -200,7 +199,7 @@ func TestTrustBundleCert(t *testing.T) { } for _, tc := range cases { t.Run(tc.desc, func(t *testing.T) { - t.Logf("%s:STATUS:Starting test case: %s", logTime, tc.desc) + t.Logf("%s:STATUS:Starting test case: %s", time.Now().String(), tc.desc) //Read the serverSAN (Subject Alternative Name) from the certificate used for TLS verification. serverSAN := setup_service.ReadDecodeServerCertificate(t, tc.serverCertFile) //Build serverCertEntity for the server certificate rotation. @@ -212,7 +211,7 @@ func TestTrustBundleCert(t *testing.T) { //Create a new Cert Pool and add the certs from the trust bundle. pkcs7certs, pkcs7data, err := setup_service.Loadpkcs7TrustBundle(tc.trustBundleFile) if err != nil { - t.Fatalf("%s:STATUS:failed to load trust bundle: %v", logTime, err) + t.Fatalf("%s:STATUS:failed to load trust bundle: %v", time.Now().String(), err) } newCaCert := x509.NewCertPool() for _, c := range pkcs7certs { @@ -223,18 +222,18 @@ func TestTrustBundleCert(t *testing.T) { //Load Client certificate. newClientCert, err := tls.LoadX509KeyPair(tc.clientCertFile, tc.clientKeyFile) if err != nil { - t.Fatalf("%s:STATUS:Failed to load client cert:%v", logTime, err) + t.Fatalf("%s:STATUS:Failed to load client cert:%v", time.Now().String(), err) } if tc.newTLScreds { - t.Logf("%s:STATUS:%s:Creating new TLS credentials for client connection.", logTime, tc.desc) + t.Logf("%s:STATUS:%s:Creating new TLS credentials for client connection.", time.Now().String(), tc.desc) //Load the prior client keypair for new client TLS credentials. prevClientCert, err := tls.LoadX509KeyPair(prevClientCertFile, prevClientKeyFile) if err != nil { - t.Fatalf("%s:STATUS:%s:Failed to load previous client cert: %v", logTime, tc.desc, err) + t.Fatalf("%s:STATUS:%s:Failed to load previous client cert: %v", time.Now().String(), tc.desc, err) } oldPkcs7certs, oldPkcs7data, err := setup_service.Loadpkcs7TrustBundle(prevTrustBundleFile) if err != nil { - t.Fatalf("%s:STATUS:%sFailed to load previous trust bundle,data %v with %v", logTime, tc.desc, oldPkcs7data, err) + t.Fatalf("%s:STATUS:%sFailed to load previous trust bundle,data %v with %v", time.Now().String(), tc.desc, oldPkcs7data, err) } //Create a old set of Cert Pool and append the certs from previous trust bundle. prevCaCert := x509.NewCertPool() @@ -243,7 +242,7 @@ func TestTrustBundleCert(t *testing.T) { } //Before rotation,validation of all services with existing certificates. if result := setup_service.ServicesValidationCheck(t, prevCaCert, expectedResult, serverSAN, serverAddr, username, password, prevClientCert, tc.mismatch); !result { - t.Fatalf("%s:STATUS:%s:service validation failed before rotate- got %v, want %v.", logTime, tc.desc, result, expectedResult) + t.Fatalf("%s:STATUS:%s:service validation failed before rotate- got %v, want %v.", time.Now().String(), tc.desc, result, expectedResult) } //Retrieve the connection with previous TLS credentials for certz rotation. conn := setup_service.CreateNewDialOption(t, prevClientCert, prevCaCert, serverSAN, username, password, serverAddr) @@ -251,20 +250,20 @@ func TestTrustBundleCert(t *testing.T) { certzClient = certzpb.NewCertzClient(conn) gnmiClient = gnmi.NewGNMIClient(conn) } else { - t.Logf("%s:STATUS:%s:Using existing TLS credentials for client connection in first iteration.", logTime, tc.desc) + t.Logf("%s:STATUS:%s:Using existing TLS credentials for client connection in first iteration.", time.Now().String(), tc.desc) } //Initiate trustbundle rotation. t.Logf("STATUS:%s Initiating Certz rotation with server cert: %s and trust bundle: %s.", tc.desc, tc.serverCertFile, tc.trustBundleFile) if success := setup_service.CertzRotate(ctx, t, newCaCert, certzClient, gnmiClient, newClientCert, dut, username, password, serverSAN, serverAddr, testProfile, tc.newTLScreds, tc.mismatch, tc.scale, &serverCertEntity, &trustBundleEntity); !success { - t.Fatalf("%s:STATUS: %s:CertzRotation failed.", logTime, tc.desc) + t.Fatalf("%s:STATUS: %s:CertzRotation failed.", time.Now().String(), tc.desc) } - t.Logf("%s:STATUS:%s: TrustBundle rotation completed!", logTime, tc.desc) + t.Logf("%s:STATUS:%s: TrustBundle rotation completed!", time.Now().String(), tc.desc) //Post rotate validation of all services. t.Run("Verification of new connection after successful trustBundle rotation", func(t *testing.T) { if result := setup_service.ServicesValidationCheck(t, newCaCert, expectedResult, serverSAN, serverAddr, username, password, newClientCert, tc.mismatch); !result { t.Fatalf("STATUS:%s:service validation failed after rotate- got %v, want %v.", tc.desc, result, expectedResult) } - t.Logf("%s:STATUS:%s:service validation done!", logTime, tc.desc) + t.Logf("%s:STATUS:%s:service validation done!", time.Now().String(), tc.desc) }) //Archiving previous client cert/key and trustbundle. prevClientCertFile = tc.clientCertFile @@ -272,10 +271,10 @@ func TestTrustBundleCert(t *testing.T) { prevTrustBundleFile = tc.trustBundleFile }) } - t.Logf("%s:STATUS:Cleanup of test data.", logTime) + t.Logf("%s:STATUS:Cleanup of test data.", time.Now().String()) //Cleanup of test data. if err := setup_service.TestdataMakeCleanup(t, dirPath, timeOutVar, "./cleanup.sh"); err != nil { - t.Logf("%s:STATUS:Cleanup of testdata certificates failed!: %v", logTime, err) + t.Logf("%s:STATUS:Cleanup of testdata certificates failed!: %v", time.Now().String(), err) } - t.Logf("%s:STATUS:Test completed!", logTime) + t.Logf("%s:STATUS:Test completed!", time.Now().String()) }