Skip to content

Commit 5751161

Browse files
askyrieSongZhen0704
authored andcommitted
feat: cloud aws support rds and redis
1 parent 80c3282 commit 5751161

8 files changed

Lines changed: 280 additions & 54 deletions

File tree

server/controller/cloud/aws/aws.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ type Aws struct {
5858
vmIDToPrivateIP map[string]string
5959
vpcIDToLcuuid map[string]string
6060
instanceIDToPrimaryIP map[string]string
61+
subnetIDToVPCAZLcuuid map[string][2]string
6162
publicIPToVinterface map[string]model.VInterface
6263
credential awsconfig.LoadOptionsFunc
6364
}
@@ -119,6 +120,7 @@ func NewAws(orgID int, domain metadbmodel.Domain, cfg cloudconfig.CloudConfig) (
119120
vmIDToPrivateIP: map[string]string{},
120121
vpcIDToLcuuid: map[string]string{},
121122
instanceIDToPrimaryIP: map[string]string{},
123+
subnetIDToVPCAZLcuuid: map[string][2]string{},
122124
publicIPToVinterface: map[string]model.VInterface{},
123125
}, nil
124126
}
@@ -194,6 +196,7 @@ func (a *Aws) GetCloudData() (model.Resource, error) {
194196
a.azLcuuidMap = map[string]int{}
195197
a.vpcIDToLcuuid = map[string]string{}
196198
a.instanceIDToPrimaryIP = map[string]string{}
199+
a.subnetIDToVPCAZLcuuid = map[string][2]string{}
197200

198201
vpcs, err := a.getVPCs(ec2Client)
199202
if err != nil {
@@ -244,6 +247,18 @@ func (a *Aws) GetCloudData() (model.Resource, error) {
244247
}
245248
resource.VMs = append(resource.VMs, vms...)
246249

250+
rdsInstances, err := a.getRDSInstances(region)
251+
if err != nil {
252+
return model.Resource{}, err
253+
}
254+
resource.RDSInstances = append(resource.RDSInstances, rdsInstances...)
255+
256+
redisInstances, err := a.getRedisInstances(region)
257+
if err != nil {
258+
return model.Resource{}, err
259+
}
260+
resource.RedisInstances = append(resource.RedisInstances, redisInstances...)
261+
247262
lbs, lbListeners, lbTargetServers, err := a.getLoadBalances(region)
248263
if err != nil {
249264
return model.Resource{}, err

server/controller/cloud/aws/lb.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ func (a *Aws) getLoadBalances(region string) ([]model.LB, []model.LBListener, []
113113
LBListenerLcuuid: listenerLcuuid,
114114
Type: common.LB_SERVER_TYPE_VM,
115115
VMLcuuid: common.GetUUIDByOrgID(a.orgID, serverInstanceID),
116-
Port: int(listener.Listener.InstancePort),
116+
Port: int(a.getInt32PointerValue(listener.Listener.InstancePort)),
117117
VPCLcuuid: vpcLcuuid,
118118
IP: ip,
119119
Protocol: a.getStringPointerValue(listener.Listener.InstanceProtocol),

server/controller/cloud/aws/network.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ func (a *Aws) getNetworks(client *ec2.Client) ([]model.Network, []model.Subnet,
8484
VPCLcuuid: vpcLcuuid,
8585
NetworkLcuuid: networkLcuuid,
8686
})
87+
a.subnetIDToVPCAZLcuuid[networkSubnetID] = [2]string{vpcLcuuid, azLcuuid}
8788

8889
routerID, ok := a.vpcOrSubnetToRouter[networkSubnetID]
8990
if !ok {

server/controller/cloud/aws/rds.go

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
/*
2+
* Copyright (c) 2024 Yunshan Networks
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package aws
18+
19+
import (
20+
"context"
21+
22+
"github.com/aws/aws-sdk-go-v2/config"
23+
"github.com/aws/aws-sdk-go-v2/service/rds"
24+
"github.com/aws/aws-sdk-go-v2/service/rds/types"
25+
"github.com/deepflowio/deepflow/server/controller/cloud/model"
26+
"github.com/deepflowio/deepflow/server/controller/common"
27+
"github.com/deepflowio/deepflow/server/libs/logger"
28+
)
29+
30+
var rdsTypeEnums = map[string]int{
31+
"mysql": common.RDS_TYPE_MYSQL,
32+
"sqlserver-ex": common.RDS_TYPE_SQL_SERVER,
33+
"sqlserver-web": common.RDS_TYPE_SQL_SERVER,
34+
"sqlserver-se": common.RDS_TYPE_SQL_SERVER,
35+
"sqlserver-ee": common.RDS_TYPE_SQL_SERVER,
36+
"postgres": common.RDS_TYPE_PSQL,
37+
"mariadb": common.RDS_TYPE_MARIADB,
38+
"oracle-ee": common.RDS_TYPE_ORACLE,
39+
"oracle-se2": common.RDS_TYPE_ORACLE,
40+
}
41+
42+
func (a *Aws) getRDSInstances(region string) ([]model.RDSInstance, error) {
43+
log.Debug("get rds instances starting", logger.NewORGPrefix(a.orgID))
44+
var rdss []model.RDSInstance
45+
46+
rdsClientConfig, err := config.LoadDefaultConfig(context.TODO(), a.credential, config.WithRegion(region), config.WithHTTPClient(a.httpClient))
47+
if err != nil {
48+
log.Error("client config failed (%s)", err.Error(), logger.NewORGPrefix(a.orgID))
49+
return []model.RDSInstance{}, err
50+
}
51+
52+
var retRDS []types.DBInstance
53+
var marker string
54+
var maxRecords int32 = 100
55+
for {
56+
var input *rds.DescribeDBInstancesInput
57+
if marker == "" {
58+
input = &rds.DescribeDBInstancesInput{MaxRecords: &maxRecords}
59+
} else {
60+
input = &rds.DescribeDBInstancesInput{MaxRecords: &maxRecords, Marker: &marker}
61+
}
62+
result, err := rds.NewFromConfig(rdsClientConfig).DescribeDBInstances(context.TODO(), input)
63+
if err != nil {
64+
log.Errorf("rds request aws api error: (%s)", err.Error(), logger.NewORGPrefix(a.orgID))
65+
return []model.RDSInstance{}, err
66+
}
67+
retRDS = append(retRDS, result.DBInstances...)
68+
if result.Marker == nil {
69+
break
70+
}
71+
marker = *result.Marker
72+
}
73+
for _, rds := range retRDS {
74+
rdsID := a.getStringPointerValue(rds.DbiResourceId)
75+
rdsLcuuid := common.GetUUIDByOrgID(a.orgID, rdsID)
76+
rdsName := a.getStringPointerValue(rds.DBInstanceIdentifier)
77+
rdsEngine := a.getStringPointerValue(rds.Engine)
78+
rdsType, ok := rdsTypeEnums[rdsEngine]
79+
if !ok {
80+
log.Infof("rds (%s) engine (%s) is not supported", rdsName, rdsEngine, logger.NewORGPrefix(a.orgID))
81+
continue
82+
}
83+
azLcuuid := common.GetUUIDByOrgID(a.orgID, a.getStringPointerValue(rds.AvailabilityZone))
84+
vpcLcuuid := common.GetUUIDByOrgID(a.orgID, a.getStringPointerValue(rds.DBSubnetGroup.VpcId))
85+
var rdsState = common.RDS_STATE_RUNNING
86+
if a.getStringPointerValue(rds.DBInstanceStatus) != "available" {
87+
rdsState = common.RDS_STATE_RESTORING
88+
}
89+
rdss = append(rdss, model.RDSInstance{
90+
Lcuuid: rdsLcuuid,
91+
Name: rdsName,
92+
Label: rdsID,
93+
State: rdsState,
94+
Type: rdsType,
95+
Series: common.RDS_SERIES_BASIC,
96+
Version: a.getStringPointerValue(rds.EngineVersion),
97+
Model: common.RDS_MODEL_PRIMARY,
98+
VPCLcuuid: vpcLcuuid,
99+
AZLcuuid: azLcuuid,
100+
RegionLcuuid: a.regionLcuuid,
101+
})
102+
a.azLcuuidMap[azLcuuid] = 0
103+
}
104+
log.Debug("get rds instances complete", logger.NewORGPrefix(a.orgID))
105+
return rdss, nil
106+
}
Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
/*
2+
* Copyright (c) 2024 Yunshan Networks
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package aws
18+
19+
import (
20+
"context"
21+
22+
"github.com/aws/aws-sdk-go-v2/config"
23+
"github.com/aws/aws-sdk-go-v2/service/elasticache"
24+
"github.com/aws/aws-sdk-go-v2/service/elasticache/types"
25+
"github.com/deepflowio/deepflow/server/controller/cloud/model"
26+
"github.com/deepflowio/deepflow/server/controller/common"
27+
"github.com/deepflowio/deepflow/server/libs/logger"
28+
)
29+
30+
func (a *Aws) getRedisInstances(region string) ([]model.RedisInstance, error) {
31+
log.Debug("get redis instances starting", logger.NewORGPrefix(a.orgID))
32+
var rediss []model.RedisInstance
33+
34+
redisClientConfig, err := config.LoadDefaultConfig(context.TODO(), a.credential, config.WithRegion(region), config.WithHTTPClient(a.httpClient))
35+
if err != nil {
36+
log.Error("client config failed (%s)", err.Error(), logger.NewORGPrefix(a.orgID))
37+
return []model.RedisInstance{}, err
38+
}
39+
40+
var retRedis []types.ServerlessCache
41+
var nextToken string
42+
var maxResults int32 = 100
43+
for {
44+
var input *elasticache.DescribeServerlessCachesInput
45+
if nextToken == "" {
46+
input = &elasticache.DescribeServerlessCachesInput{MaxResults: &maxResults}
47+
} else {
48+
input = &elasticache.DescribeServerlessCachesInput{MaxResults: &maxResults, NextToken: &nextToken}
49+
}
50+
result, err := elasticache.NewFromConfig(redisClientConfig).DescribeServerlessCaches(context.TODO(), input)
51+
if err != nil {
52+
log.Errorf("redis request aws api error: (%s)", err.Error(), logger.NewORGPrefix(a.orgID))
53+
return []model.RedisInstance{}, err
54+
}
55+
retRedis = append(retRedis, result.ServerlessCaches...)
56+
if result.NextToken == nil {
57+
break
58+
}
59+
nextToken = *result.NextToken
60+
}
61+
for _, redis := range retRedis {
62+
redisName := a.getStringPointerValue(redis.ServerlessCacheName)
63+
redisLcuuid := common.GetUUIDByOrgID(a.orgID, redisName)
64+
var redisState int
65+
if a.getStringPointerValue(redis.Status) == "available" {
66+
redisState = common.REDIS_STATE_RUNNING
67+
}
68+
var vpcLcuuid string
69+
var azLcuuid string
70+
for _, subnetID := range redis.SubnetIds {
71+
nets, ok := a.subnetIDToVPCAZLcuuid[subnetID]
72+
if ok {
73+
vpcLcuuid = nets[0]
74+
azLcuuid = nets[1]
75+
break
76+
}
77+
78+
}
79+
if vpcLcuuid == "" || azLcuuid == "" {
80+
log.Infof("redis instance (%s) vpc or az not found, subnet IDs: %v", redisName, redis.SubnetIds, logger.NewORGPrefix(a.orgID))
81+
continue
82+
}
83+
rediss = append(rediss, model.RedisInstance{
84+
Lcuuid: redisLcuuid,
85+
Name: redisName,
86+
Label: redisName,
87+
State: redisState,
88+
Version: "Redis " + a.getStringPointerValue(redis.FullEngineVersion),
89+
AZLcuuid: azLcuuid,
90+
VPCLcuuid: vpcLcuuid,
91+
RegionLcuuid: a.regionLcuuid,
92+
})
93+
a.azLcuuidMap[azLcuuid] = 0
94+
}
95+
log.Debug("get redis instances complete", logger.NewORGPrefix(a.orgID))
96+
return rediss, nil
97+
}

server/controller/common/const.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -527,6 +527,7 @@ const (
527527
RDS_TYPE_PPAS = 3
528528
RDS_TYPE_PSQL = 4 // PostgreSQL
529529
RDS_TYPE_MARIADB = 5
530+
RDS_TYPE_ORACLE = 6 // oracle
530531

531532
RDS_STATE_RUNNING = 1
532533
RDS_STATE_RESTORING = 2
@@ -542,7 +543,8 @@ const (
542543
)
543544

544545
const (
545-
REDIS_STATE_RUNNING = 1
546+
REDIS_STATE_RUNNING = 1
547+
REDIS_STATE_RECOVERING = 2
546548
)
547549

548550
const (

server/go.mod

Lines changed: 19 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -39,13 +39,15 @@ require (
3939
github.com/Workiva/go-datastructures v1.0.53
4040
github.com/agiledragon/gomonkey/v2 v2.8.0
4141
github.com/aliyun/alibaba-cloud-sdk-go v1.61.1633
42-
github.com/aws/aws-sdk-go-v2 v1.17.3
43-
github.com/aws/aws-sdk-go-v2/config v1.17.8
44-
github.com/aws/aws-sdk-go-v2/credentials v1.12.21
45-
github.com/aws/aws-sdk-go-v2/service/ec2 v1.63.1
46-
github.com/aws/aws-sdk-go-v2/service/eks v1.26.0
47-
github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.14.18
48-
github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.18.20
42+
github.com/aws/aws-sdk-go-v2 v1.39.0
43+
github.com/aws/aws-sdk-go-v2/config v1.31.8
44+
github.com/aws/aws-sdk-go-v2/credentials v1.18.12
45+
github.com/aws/aws-sdk-go-v2/service/ec2 v1.253.0
46+
github.com/aws/aws-sdk-go-v2/service/eks v1.73.3
47+
github.com/aws/aws-sdk-go-v2/service/elasticache v1.50.3
48+
github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.33.4
49+
github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.50.4
50+
github.com/aws/aws-sdk-go-v2/service/rds v1.107.0
4951
github.com/baidubce/bce-sdk-go v0.9.141
5052
github.com/bitly/go-simplejson v0.5.0
5153
github.com/bxcodec/faker/v3 v3.8.0
@@ -159,15 +161,16 @@ require (
159161
github.com/KyleBanks/depth v1.2.1 // indirect
160162
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
161163
github.com/aws/aws-sdk-go v1.44.37 // indirect
162-
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17 // indirect
163-
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 // indirect
164-
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 // indirect
165-
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24 // indirect
166-
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17 // indirect
167-
github.com/aws/aws-sdk-go-v2/service/sso v1.11.23 // indirect
168-
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.6 // indirect
169-
github.com/aws/aws-sdk-go-v2/service/sts v1.16.19 // indirect
170-
github.com/aws/smithy-go v1.13.5 // indirect
164+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.7 // indirect
165+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.7 // indirect
166+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.7 // indirect
167+
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
168+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 // indirect
169+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.7 // indirect
170+
github.com/aws/aws-sdk-go-v2/service/sso v1.29.3 // indirect
171+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.4 // indirect
172+
github.com/aws/aws-sdk-go-v2/service/sts v1.38.4 // indirect
173+
github.com/aws/smithy-go v1.23.0 // indirect
171174
github.com/beorn7/perks v1.0.1 // indirect
172175
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
173176
github.com/bytedance/sonic/loader v0.3.0 // indirect

0 commit comments

Comments
 (0)