Skip to content

Commit 6707b38

Browse files
committed
Add the unmarshal command
1 parent edb50f8 commit 6707b38

12 files changed

Lines changed: 233 additions & 234 deletions

File tree

.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,5 @@
33
.DS_Store
44
analysis.txt
55
PLAN.MD
6-
etcdctl+
6+
etcdctl+
7+
bin/etcdctl+

README.md

Lines changed: 57 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,26 +3,59 @@ etcd is generally used to store system metadata or service discovery, and is sui
33

44
When testing the stability of the system, it may be necessary to pay attention to the size distribution of the data currently stored in etcd by the system. That's why this project came about.
55

6+
# 🌟 New Function: unmarshal
7+
8+
**implement proto.Unmarshal byte array through proto source file**
9+
10+
Generally speaking, we store some system meta formation in etcd, and the stored pseudocode is:
11+
12+
```go
13+
segBytes, _ := proto.Marshal(*foopb.SystemInfo{})
14+
etcdclient.put("foo/system", segBytes)
15+
```
16+
17+
When we query, we cannot clearly see the value in the struct. Of course, it can be easily implemented through code. You only need to import the relevant pb file and then call the proto.Unmarshal method to see the clear value. But many times we may not have the environment, or writing this part of the code will waste a little time.
18+
19+
The unmarshal instruction will solve this trouble. You only need to copy the proto source file to quickly view the value in the struct. The example:
20+
21+
```
22+
➜ etcdctl+ unmarshal --key by-dev/meta/channelwatch/4/by-dev-rootcoord-dml_0_445337303926193462v0 --import-path ../birdwatcher/proto/v2.2 --proto ../birdwatcher/proto/v2.2/data_coord.proto --full-message-name milvus.protov2.data.ChannelWatchInfo
23+
24+
vchan: collectionID:445337303926193462 channelName:"by-dev-rootcoord-dml_0_445337303926193462v0" seek_position:<channel_name:"by-dev-rootcoord-dml_0" msgID:"\010?\020\223\261\002\030\000 \000" msgGroup:"datanode-3-by-dev-rootcoord-dml_0_445337303926193462v0-true" timestamp:445342561987461121> flushedSegmentIds:445337303926393472
25+
startTs: 1698912217
26+
state: 3
27+
timeoutTs: 0
28+
schema: name:"foo" description:"hello_milvus is the simplest demo to introduce the APIs" fields:<fieldID:100 name:"pk" is_primary_key:true data_type:VarChar type_params:<key:"max_length" value:"100">> fields:<fieldID:101 name:"random" data_type:Double> fields:<fieldID:102 name:"embeddings" data_type:FloatVector type_params:<key:"dim" value:"8">> fields:<name:"RowID" description:"row id" data_type:Int64> fields:<fieldID:1 name:"Timestamp" description:"time stamp" data_type:Int64>
29+
progress: 0
30+
```
31+
632
<details>
733
<summary><h1>Getting started</h1></summary>
834

935
## Getting the source code
36+
1037
Clone this code repository
38+
1139
```shell
1240
$ git clone https://github.com/SimFG/etcd-analysis.git
1341
```
42+
1443
## Build
44+
1545
Compile code into executable
46+
1647
```shell
1748
$ go build -o etcdctl+
1849
```
50+
1951
## Usage
52+
2053
Get help with functions
54+
2155
```shell
2256
$ etcdctl+ distribute -h
2357
```
24-
## Auto complete config
25-
Download the [etcdctl+.ts](ts/etcdctl+.ts), and [config it](https://simfg.github.io/fig). You can also use the [etcdctl.ts](https://simfg.github.io/etcdctl.ts) config the `etcdctl` command.
58+
2659
</details>
2760

2861
<details open>
@@ -34,10 +67,12 @@ Download the [etcdctl+.ts](ts/etcdctl+.ts), and [config it](https://simfg.github
3467
3. **find** Get key based on certain characters
3568
4. **leader** Get the leader node info
3669
5. **clear** Clear all the etcd data
37-
6. **decode** Decode the etcd value that is encoded
70+
6. **decode** Base64Decode the etcd value that is encoded
3871
7. **rename** Rename the etcd data key
72+
8. **unmarshal** Implement proto.Unmarshal byte array through proto source file
3973

4074
## distribute
75+
4176
View data distribution in etcd according to the `key` size , `value` size or `key + value` size by setting the `type` command param.
4277

4378
![distribute.gif](pic/20230225-150850.gif)
@@ -99,7 +134,9 @@ Kv List
99134
```
100135

101136
## find
137+
102138
Get key based on certain characters
139+
103140
```shell
104141
$ ./etcdctl+ find --key=index
105142
Kv List
@@ -112,7 +149,9 @@ Kv List
112149
```
113150

114151
## leader
152+
115153
Get the leader node info
154+
116155
```shell
117156
$ etcdctl+ leader
118157

@@ -121,13 +160,28 @@ ClientUrls: [http://127.0.0.1:2379]
121160
```
122161

123162
## clear
163+
124164
Clear all etcd data
165+
125166
```shell
126167
$ etcdctl+ clear
127168

128169
Clear All Data, (Y/n):
129170
```
130171

172+
## unmarshal
173+
174+
Implement proto.Unmarshal byte array through proto source file.
175+
176+
```
177+
$ etcdctl+ unmarshal --key by-dev/meta/channelwatch/4/by-dev-rootcoord-dml_0_445337303926193462v0 --import-path ../birdwatcher/proto/v2.2 --proto ../birdwatcher/proto/v2.2/data_coord.proto --full-message-name milvus.protov2.data.ChannelWatchInfo
178+
```
179+
180+
- key: the etcd full key
181+
- import-path: all proto directory
182+
- proto: the proto file path where the message is located
183+
- full-message-name: the full message name, usually a combination of proto package name and message
184+
131185
</details>
132186

133187
# All contributors

bin/etcdctl+

-17.5 MB
Binary file not shown.

cmd/decode_cmd.go

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,11 @@ package cmd
33
import (
44
"encoding/base64"
55
"fmt"
6+
67
"github.com/spf13/cobra"
78
)
89

9-
var (
10-
encodedValue string
11-
)
10+
var encodedValue string
1211

1312
func NewDecodeCmd() *cobra.Command {
1413
cmd := &cobra.Command{
@@ -28,5 +27,4 @@ func decodeFunc(cmd *cobra.Command, args []string) {
2827
} else {
2928
fmt.Println("decode value:\n", string(v))
3029
}
31-
3230
}

cmd/find_cmd.go

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,19 @@ import (
44
"bytes"
55
"encoding/base64"
66
"fmt"
7+
"io"
8+
"os"
9+
"strings"
10+
711
"github.com/SimFG/etcd-analysis/core"
812
"github.com/spf13/cobra"
913
"go.etcd.io/etcd/api/v3/mvccpb"
1014
clientv3 "go.etcd.io/etcd/client/v3"
11-
"io"
12-
"os"
13-
"strings"
1415
)
1516

1617
var (
1718
findKey = ""
19+
findPrefix = ""
1820
containValue = false
1921
findLimit = 10
2022
)
@@ -27,14 +29,15 @@ func NewFindCmd() *cobra.Command {
2729
}
2830

2931
cmd.Flags().StringVar(&findKey, "key", "", "Show the data like the key")
32+
cmd.Flags().StringVar(&findPrefix, "prefix", "", "Show the data like the prefix")
3033
cmd.Flags().BoolVar(&containValue, "value", false, "Show the value or not")
3134
cmd.Flags().IntVar(&findLimit, "limit", 10, "The limit of the show keys")
3235
return cmd
3336
}
3437

3538
func findFunc(cmd *cobra.Command, args []string) {
3639
core.InitClient()
37-
resp, datac := core.GetAllData()
40+
resp, datac := core.GetDataWithPrefix(findPrefix)
3841
appendBufferForFind(resp, datac, os.Stdout)
3942
}
4043

cmd/root_cmd.go

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,10 @@ import (
99
cobracompletefig "github.com/withfig/autocomplete-tools/integrations/cobra"
1010
)
1111

12-
var (
13-
rootCmd = &cobra.Command{
14-
Use: "etcdctl+",
15-
Short: "etcd data analysis tool",
16-
}
17-
)
12+
var rootCmd = &cobra.Command{
13+
Use: "etcdctl+",
14+
Short: "etcd data analysis tool",
15+
}
1816

1917
func Start() {
2018
if err := rootCmd.Execute(); err != nil {
@@ -41,5 +39,6 @@ func init() {
4139
rootCmd.AddCommand(NewFindCmd())
4240
rootCmd.AddCommand(NewDecodeCmd())
4341
rootCmd.AddCommand(NewRenameCmd())
42+
rootCmd.AddCommand(NewUnmarshalCmd())
4443
rootCmd.AddCommand(cobracompletefig.CreateCompletionSpecCommand())
4544
}

cmd/unmarsha_cmd.go

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
package cmd
2+
3+
import (
4+
"errors"
5+
"fmt"
6+
7+
"github.com/SimFG/etcd-analysis/core"
8+
"github.com/golang/protobuf/proto"
9+
"github.com/jhump/protoreflect/desc"
10+
"github.com/jhump/protoreflect/desc/protoparse"
11+
"github.com/jhump/protoreflect/dynamic"
12+
"github.com/spf13/cobra"
13+
)
14+
15+
var (
16+
unmarshallKey string
17+
unmarshalImportPaths []string
18+
unmarshalFileNames []string
19+
unmarshalFullMessageName string
20+
)
21+
22+
func NewUnmarshalCmd() *cobra.Command {
23+
cmd := &cobra.Command{
24+
Use: "unmarshal",
25+
Short: "unmarshal the etcd value",
26+
Run: unmarshalFunc,
27+
}
28+
29+
cmd.Flags().StringVar(&unmarshallKey, "key", "", "the proto.marshal value of the full key")
30+
cmd.Flags().StringArrayVar(&unmarshalImportPaths, "import-path", []string{}, "the proto.marshal value of the full key")
31+
cmd.Flags().StringArrayVar(&unmarshalFileNames, "proto", []string{}, "the proto.marshal value of the full key")
32+
cmd.Flags().StringVar(&unmarshalFullMessageName, "full-message-name", "", "the proto.marshal value of the full key")
33+
34+
return cmd
35+
}
36+
37+
func unmarshalFunc(cmd *cobra.Command, args []string) {
38+
client := core.InitClient()
39+
resp, err := core.EtcdGet(client, unmarshallKey)
40+
if err != nil {
41+
core.Exit(err)
42+
}
43+
if len(resp.Kvs) == 0 {
44+
core.Exit(errors.New("not found the key"))
45+
}
46+
bytesValue := resp.Kvs[0].Value
47+
decodeBytes(bytesValue)
48+
return
49+
}
50+
51+
func decodeBytes(bytesValue []byte) {
52+
fileNames, err := protoparse.ResolveFilenames(unmarshalImportPaths, unmarshalFileNames...)
53+
if err != nil {
54+
fmt.Println("counld not resolve file names")
55+
core.Exit(err)
56+
}
57+
p := protoparse.Parser{
58+
ImportPaths: unmarshalImportPaths,
59+
InferImportPaths: len(unmarshalImportPaths) == 0,
60+
IncludeSourceCodeInfo: true,
61+
}
62+
63+
fds, err := p.ParseFiles(fileNames...)
64+
if err != nil {
65+
fmt.Println("could not parse given files")
66+
core.Exit(err)
67+
}
68+
var messageType *desc.MessageDescriptor
69+
for _, protoDesc := range fds {
70+
messageType = protoDesc.FindMessage(unmarshalFullMessageName)
71+
if messageType != nil {
72+
break
73+
}
74+
}
75+
if messageType == nil {
76+
fmt.Println("could not find message type, please check the message name is full, name: " + unmarshalFullMessageName)
77+
return
78+
}
79+
message := dynamic.NewMessage(messageType)
80+
err = proto.Unmarshal(bytesValue, message)
81+
if err != nil {
82+
fmt.Println("could not unmarshal bytes")
83+
core.Exit(err)
84+
}
85+
fmt.Println()
86+
fields := message.GetKnownFields()
87+
for _, field := range fields {
88+
fieldValue := message.GetField(field)
89+
fmt.Printf("%s: %v\n", field.GetName(), fieldValue)
90+
}
91+
}

core/data_source.go

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,7 @@ const (
1313
readCount = 1000
1414
)
1515

16-
var (
17-
client *clientv3.Client
18-
)
16+
var client *clientv3.Client
1917

2018
func InitClient() *clientv3.Client {
2119
connEndpoints := []string{"127.0.0.1:2379"}
@@ -51,19 +49,31 @@ func InitClient() *clientv3.Client {
5149
return client
5250
}
5351

54-
func GetAllData() (*clientv3.GetResponse, <-chan []*mvccpb.KeyValue) {
52+
func GetDataWithPrefix(prefix string) (*clientv3.GetResponse, <-chan []*mvccpb.KeyValue) {
5553
c := make(chan []*mvccpb.KeyValue, 10)
5654

5755
getFunc := func(start string, count int64) *clientv3.GetResponse {
58-
resp, err := EtcdGet(client, start, clientv3.WithFromKey(),
56+
opts := []clientv3.OpOption{
5957
clientv3.WithSerializable(),
60-
//clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend),
61-
clientv3.WithLimit(count))
58+
clientv3.WithLimit(count),
59+
}
60+
if prefix != "" {
61+
opts = append(opts, clientv3.WithPrefix())
62+
} else {
63+
opts = append(opts, clientv3.WithFromKey())
64+
}
65+
resp, err := EtcdGet(client, start, opts...)
6266
if err != nil {
6367
Exit(err)
6468
}
6569
return resp
6670
}
71+
if prefix != "" {
72+
resp := getFunc(prefix, 0)
73+
c <- resp.Kvs
74+
close(c)
75+
return resp, c
76+
}
6777
resp := getFunc(EmptyChar(), 2)
6878
c <- resp.Kvs
6979

@@ -87,3 +97,7 @@ func GetAllData() (*clientv3.GetResponse, <-chan []*mvccpb.KeyValue) {
8797

8898
return resp, c
8999
}
100+
101+
func GetAllData() (*clientv3.GetResponse, <-chan []*mvccpb.KeyValue) {
102+
return GetDataWithPrefix("")
103+
}

0 commit comments

Comments
 (0)