Skip to content

Commit e9d0fe9

Browse files
author
Rohit-PX
committed
API for exporting stats
Signed-off-by: Rohit-PX <rkulkarni@purestorage.com>
1 parent c14c9b4 commit e9d0fe9

7 files changed

Lines changed: 462 additions & 2 deletions

File tree

‎pkg/utils/export_stats.go‎

Lines changed: 218 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,218 @@
1+
package utils
2+
3+
import (
4+
"bytes"
5+
"encoding/json"
6+
"fmt"
7+
"io/ioutil"
8+
"net/http"
9+
"strconv"
10+
"strings"
11+
"time"
12+
13+
storkv1 "github.com/libopenstorage/stork/pkg/apis/stork/v1alpha1"
14+
"github.com/sirupsen/logrus"
15+
)
16+
17+
const (
18+
AetosStatsURL = "http://aetos.pwx.purestorage.com/dashboard/stats?stats_type=MigrationDemoFinal&limit=100"
19+
)
20+
21+
var StorkVersion string
22+
var PortworxVersion string
23+
24+
type StatsExportType struct {
25+
id OidType `json: "_id",omitempty`
26+
Name string `json: "name",omitempty`
27+
Product string `json: "product",omitempty`
28+
Version string `json: "version",omitempty`
29+
StatsType string `json: "statsType",omitempty`
30+
Data MigrationStatsType `json: "data",omitempty`
31+
}
32+
33+
type MigrationStatsType struct {
34+
CreatedOn string `json: "created",omitempty`
35+
TotalNumberOfVolumes json.Number `json: "totalNumberOfVolumes",omitempty`
36+
NumOfMigratedVolumes json.Number `json: "numOfMigratedVolumes",omitempty`
37+
TotalNumberOfResources json.Number `json: "totalNumberOfResources",omitempty`
38+
NumOfMigratedResources json.Number `json: "numOfMigratedResources",omitempty`
39+
TotalBytesMigrated json.Number `json: "totalBytesMigrated",omitempty`
40+
ElapsedTimeForVolumeMigration string `json: "elapsedTimeForVolumeMigration",omitempty`
41+
ElapsedTimeForResourceMigration string `json: "elapsedTimeForResourceMigration",omitempty`
42+
Application string `json: "application",omitempty`
43+
StorkVersion string `json: "storkVersion",omitempty`
44+
PortworxVersion string `json: "portworxVersion",omitempty`
45+
}
46+
47+
type OidType struct {
48+
oid string `json: "$oid"`
49+
}
50+
51+
type AllStats []StatsExportType
52+
53+
func GetMigrationStatsFromAetos(url string) ([]StatsExportType, error) {
54+
client := http.Client{Timeout: time.Second * 3}
55+
req, err := http.NewRequest("GET", url, nil)
56+
if err != nil {
57+
return nil, fmt.Errorf("error querying Aetos for stats: %v", err)
58+
}
59+
60+
req.Header.Add("Content-Type", "application/json")
61+
62+
q := req.URL.Query()
63+
q.Add("format", "json")
64+
req.URL.RawQuery = q.Encode()
65+
resp, err := client.Do(req)
66+
if err != nil {
67+
return nil, fmt.Errorf("error querying Aetos metadata: %v", err)
68+
}
69+
70+
if resp.StatusCode != 200 {
71+
return nil, fmt.Errorf("error querying Aetos metadata: Code %d returned for url %s", resp.StatusCode, req.URL)
72+
}
73+
body, err := ioutil.ReadAll(resp.Body)
74+
if err != nil {
75+
return nil, fmt.Errorf("error querying Aetos metadata: %v", err)
76+
}
77+
if len(body) == 0 {
78+
return nil, fmt.Errorf("error querying Aetos metadata: Empty response")
79+
}
80+
81+
//fmt.Printf("\nBody of response: %v\n\n", string(body))
82+
data := AllStats{}
83+
err = json.Unmarshal(body, &data)
84+
if err != nil {
85+
return nil, fmt.Errorf("error parsing Aetos metadata: %v", err)
86+
}
87+
88+
defer func() {
89+
err := resp.Body.Close()
90+
if err != nil {
91+
logrus.Errorf("Error closing body when getting Aetos data: %v", err)
92+
}
93+
}()
94+
return data, nil
95+
}
96+
97+
func WriteMigrationStatsToAetos(data StatsExportType) error {
98+
body, err := json.Marshal(data)
99+
if err != nil {
100+
return fmt.Errorf("failed to marshal: %v", err)
101+
}
102+
resp, err := http.Post("http://aetos.pwx.purestorage.com/dashboard/stats", "application/json", bytes.NewBuffer(body))
103+
if err != nil {
104+
return fmt.Errorf("post request to Aetos failed: %v", err)
105+
}
106+
defer resp.Body.Close()
107+
if resp.StatusCode == http.StatusCreated || resp.StatusCode == http.StatusOK {
108+
body, err := ioutil.ReadAll(resp.Body)
109+
if err != nil {
110+
//Failed to read response.
111+
return fmt.Errorf("response from Aetos failed: %v", err)
112+
}
113+
114+
jsonStr := string(body)
115+
logrus.Infof("Stats successfully pushed to DB. Response: ", jsonStr)
116+
} else {
117+
return fmt.Errorf("Get failed with Reponse status: %s", resp.Status)
118+
}
119+
120+
return nil
121+
}
122+
123+
func MockStat() StatsExportType {
124+
mockStat := StatsExportType{
125+
Name: "stork_integration_test",
126+
Product: "stork",
127+
StatsType: "migration_stats_mock",
128+
Version: "v1alpha1",
129+
Data: MigrationStatsType{
130+
TotalNumberOfVolumes: "1",
131+
NumOfMigratedVolumes: "1",
132+
TotalNumberOfResources: "5",
133+
NumOfMigratedResources: "1",
134+
TotalBytesMigrated: "12345",
135+
ElapsedTimeForVolumeMigration: "11.1111s",
136+
ElapsedTimeForResourceMigration: "12321.3453s",
137+
Application: "mock_app",
138+
StorkVersion: StorkVersion,
139+
PortworxVersion: PortworxVersion,
140+
},
141+
}
142+
return mockStat
143+
}
144+
145+
func NewStat() StatsExportType {
146+
newStat := StatsExportType{
147+
Name: "stork_integration_test",
148+
Product: "stork",
149+
StatsType: "MigrationDemoFinal",
150+
Version: "v1alpha1",
151+
Data: MigrationStatsType{
152+
TotalNumberOfVolumes: "",
153+
Application: "",
154+
NumOfMigratedVolumes: "",
155+
TotalNumberOfResources: "",
156+
NumOfMigratedResources: "",
157+
TotalBytesMigrated: "",
158+
ElapsedTimeForVolumeMigration: "",
159+
ElapsedTimeForResourceMigration: "",
160+
StorkVersion: StorkVersion,
161+
PortworxVersion: PortworxVersion,
162+
},
163+
}
164+
return newStat
165+
}
166+
167+
func GetExportableStatsFromMigrationObject(mig *storkv1.Migration) StatsExportType {
168+
exportStats := NewStat()
169+
170+
exportStats.Data.Application = getResourceNamesFromMigration(mig)
171+
172+
exportStats.Data.CreatedOn = (mig.GetCreationTimestamp()).Format("2006-01-02 15:04:05")
173+
174+
exportStats.Data.TotalNumberOfVolumes = json.Number(strconv.FormatUint(mig.Status.Summary.TotalNumberOfVolumes, 10))
175+
176+
exportStats.Data.NumOfMigratedVolumes = json.Number(strconv.FormatUint(mig.Status.Summary.NumberOfMigratedVolumes, 10))
177+
178+
exportStats.Data.TotalNumberOfResources = json.Number(strconv.FormatUint(mig.Status.Summary.TotalNumberOfResources, 10))
179+
180+
exportStats.Data.NumOfMigratedResources = json.Number(strconv.FormatUint(mig.Status.Summary.NumberOfMigratedResources, 10))
181+
182+
exportStats.Data.TotalBytesMigrated = json.Number(strconv.FormatUint(mig.Status.Summary.TotalBytesMigrated, 10))
183+
184+
exportStats.Data.ElapsedTimeForVolumeMigration = mig.Status.Summary.ElapsedTimeForVolumeMigration
185+
186+
exportStats.Data.ElapsedTimeForResourceMigration = mig.Status.Summary.ElapsedTimeForResourceMigration
187+
188+
PrettyStruct(exportStats)
189+
190+
return exportStats
191+
192+
}
193+
194+
func PrettyStruct(data interface{}) {
195+
val, err := json.MarshalIndent(data, "", " ")
196+
if err != nil {
197+
logrus.Panicf("Failed to prettify data")
198+
}
199+
fmt.Printf("Exportable migration data: %v", string(val))
200+
}
201+
202+
func getResourceNamesFromMigration(mig *storkv1.Migration) string {
203+
var resourceList []string
204+
for _, resource := range mig.Status.Resources {
205+
if resource.Kind == "Deployment" || resource.Kind == "StatefulSet" {
206+
resourceList = append(resourceList, resource.Name)
207+
}
208+
}
209+
if len(resourceList) > 1 {
210+
// return comma separated list of apps if there are multiple apps
211+
return strings.Join(resourceList, ",")
212+
} else if len(resourceList) == 1 {
213+
return resourceList[0]
214+
}
215+
216+
logrus.Info("App name not found for pushing to DB.")
217+
return ""
218+
}

‎test/integration_test/common_test.go‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
storkv1 "github.com/libopenstorage/stork/pkg/apis/stork/v1alpha1"
2727
"github.com/libopenstorage/stork/pkg/schedule"
2828
"github.com/libopenstorage/stork/pkg/storkctl"
29+
"github.com/libopenstorage/stork/pkg/utils"
2930
"github.com/libopenstorage/stork/pkg/version"
3031
"github.com/portworx/sched-ops/k8s/apps"
3132
"github.com/portworx/sched-ops/k8s/core"
@@ -151,10 +152,11 @@ var backupLocationPath string
151152
var genericCsiConfigMap string
152153
var externalTest bool
153154
var storkVersionCheck bool
155+
var storkVersion string
156+
var pxVersion string
154157
var cloudDeletionValidate bool
155158
var isInternalLBAws bool
156159
var pxNamespace string
157-
var storkVersion string
158160
var testrailHostname string
159161
var testrailUsername string
160162
var testrailPassword string
@@ -274,12 +276,20 @@ func setup() error {
274276
return fmt.Errorf("stork version not found in configmap: %s", cmName)
275277
}
276278
storkVersion = getStorkVersion(ver)
279+
utils.StorkVersion = ver
280+
277281
if storkVersionCheck == true {
278282
if getStorkVersion(ver) != getStorkVersion(version.Version) {
279283
return fmt.Errorf("stork version mismatch, found: %s, expected: %s", getStorkVersion(ver), getStorkVersion(version.Version))
280284
}
281285
}
282286
}
287+
stc, err := operator.Instance().ListStorageClusters("kube-system")
288+
if err != nil {
289+
logrus.Warnf("failed to list PX storage cluster during setup: %v, probably a daemonset install for portworx", err)
290+
} else {
291+
utils.PortworxVersion = oputils.GetPortworxVersion(&stc.Items[0]).Original()
292+
}
283293

284294
isInternalLBAws, err = strconv.ParseBool(os.Getenv(internalLBAws))
285295
if err == nil {

‎test/integration_test/export_stats_test.go‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
//go:build integrationtest
2+
// +build integrationtest
3+
4+
package integrationtest
5+
6+
import (
7+
"encoding/json"
8+
"fmt"
9+
"testing"
10+
11+
"github.com/stretchr/testify/require"
12+
13+
"github.com/libopenstorage/stork/pkg/utils"
14+
)
15+
16+
const (
17+
aetosStatsURL = "http://aetos.pwx.purestorage.com/dashboard/stats?stats_type=migration_stats_new&limit=100"
18+
)
19+
20+
func TestExportStatsGetStats(t *testing.T) {
21+
fmt.Println("Hello")
22+
data, err := utils.GetMigrationStatsFromAetos(aetosStatsURL)
23+
require.NoError(t, err, "Failed to get stats: %v")
24+
25+
prettyData, err := PrettyStruct(data)
26+
require.NoError(t, err, "Failed to print pretty data: %v")
27+
fmt.Println(prettyData)
28+
}
29+
30+
func TestExportStatsPushMockStats(t *testing.T) {
31+
err := utils.WriteMigrationStatsToAetos(NewStat())
32+
require.NoError(t, err, "Failed to write stats: %v")
33+
}
34+
35+
func NewStat() utils.StatsExportType {
36+
mockStat := utils.StatsExportType{
37+
Name: "stork_integration_test",
38+
Product: "stork",
39+
StatsType: "migration_stats_mock",
40+
Version: "v1alpha1",
41+
Data: utils.MigrationStatsType{
42+
TotalNumberOfVolumes: "1",
43+
NumOfMigratedVolumes: "1",
44+
TotalNumberOfResources: "5",
45+
NumOfMigratedResources: "1",
46+
TotalBytesMigrated: "12345",
47+
ElapsedTimeForVolumeMigration: "11.1111s",
48+
ElapsedTimeForResourceMigration: "12321.3453s",
49+
},
50+
}
51+
return mockStat
52+
}
53+
54+
func PrettyStruct(data interface{}) (string, error) {
55+
val, err := json.MarshalIndent(data, "", " ")
56+
if err != nil {
57+
return "", err
58+
}
59+
return string(val), nil
60+
}

0 commit comments

Comments
 (0)