Skip to content

Commit ab1572f

Browse files
ignoramousignoramous
committed
Add block-stamp version support and elementary memory profile
As part of version support for block-stamps, the code is now duplicated as the legacy code that don't support versioning lingers around now for now. Added a small bit of memory profiling in the main.go test file, and removed a couple of verbose printlns from commons.go
1 parent 65b204b commit ab1572f

4 files changed

Lines changed: 220 additions & 47 deletions

File tree

‎main.go‎

Lines changed: 79 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,12 @@ package main
22

33
import "github.com/celzero/gotrie/trie"
44
import "fmt"
5+
import "runtime"
6+
import "unsafe"
7+
import (
8+
"os"
9+
"runtime/pprof"
10+
)
511

612
//import "time"
713
//import "os"
@@ -10,41 +16,78 @@ import "fmt"
1016

1117
func main() {
1218

13-
err, FT := trie.Build("./td.txt", "./rd.txt", "./basicconfig.json", "./filetag.json")
14-
if err == nil {
15-
//[33216 32768 8192 256 4]
16-
//6IeA6ICA4oCAxIAE
17-
//res := []string{"AMI","CQT","EOK","MTF"}
18-
//usr_flag := FT.CreateUrlEncodedflag(res)
19-
//fmt.Println(usr_flag)
20-
fmt.Println(FT.Urlenc_to_flag("w4DEgAQ="))
21-
fmt.Println(FT.DNlookup("google.com", "6IeA6ICA4oCAxIAE"))
22-
/*
23-
for{
24-
reader := bufio.NewReader(os.Stdin)
25-
fmt.Print("Enter text: ")
26-
text, _ := reader.ReadString('\n')
27-
text = strings.TrimSpace(text)
28-
if(text == "exit"){break;}
29-
start := time.Now()
30-
fmt.Println(FT.DNlookup(text,usr_flag))
31-
elapsed := time.Since(start)
32-
fmt.Printf("Time Diff %s\n",elapsed)
33-
}*/
34-
} else {
35-
fmt.Println("Error at trie Build")
36-
}
37-
38-
/*err,FT := trie.Build()
39-
if(err == nil){
40-
res := []string{"DAH","ADH","BXW", "BQJ"}
41-
fmt.Println("Base64 to flag : ",FT.Urlenc_to_flag(FT.CreateUrlEncodedflag(res)))
42-
}*/
43-
44-
/*
45-
start := time.Now()
46-
trie.CheckDN1()
47-
elapsed := time.Since(start)
48-
fmt.Printf("Time Diff %s\n",elapsed)*/
19+
err, FT := trie.Build("./td", "./rank", "./basicconfig", "./blocklists")
20+
if err == nil {
21+
//[33216 32768 8192 256 4]
22+
//6IeA6ICA4oCAxIAE
23+
res := []string{"AMI","CQT","EOK","MTF"}
24+
usr_flag := FT.CreateUrlEncodedflag(res)
25+
fmt.Println(usr_flag)
26+
fmt.Println(FT.Urlenc_to_flag("w4DEgAQ="))
4927

28+
fmt.Print("1: v1<>google.com ")
29+
fmt.Println(FT.DNlookup("google.com", "6IeA6ICA4oCAxIAE"))
30+
31+
t := "77%2Bg77%2B%2F77%2B%2F77%2B%2F77%2B%2F77%2B%2F77%2B%2F77%2B%2F77%2B%2F77%2B%2F77%2B%2F77%2Bg"
32+
fmt.Print("2: v1<>amazon.com ")
33+
fmt.Println(FT.DNlookup("amazon.com", t))
34+
35+
l := "1:4P___________________________-D_"
36+
fmt.Print("3: v2<>amazon.com ")
37+
fmt.Println(FT.DNlookup("amazon.com", l))
38+
39+
PrintMemUsage()
40+
41+
fmt.Println("ft: %d, td: %d, rd: %d",
42+
unsafe.Sizeof(FT), unsafe.Sizeof(FT.GetData()), unsafe.Sizeof(FT.GetDir()))
43+
/*
44+
for{
45+
reader := bufio.NewReader(os.Stdin)
46+
fmt.Print("Enter text: ")
47+
text, _ := reader.ReadString('\n')
48+
text = strings.TrimSpace(text)
49+
if(text == "exit"){break;}
50+
start := time.Now()
51+
fmt.Println(FT.DNlookup(text,usr_flag))
52+
elapsed := time.Since(start)
53+
fmt.Printf("Time Diff %s\n",elapsed)
54+
}*/
55+
} else {
56+
fmt.Println("Error at trie Build")
57+
}
58+
59+
/*err,FT := trie.Build()
60+
if(err == nil){
61+
res := []string{"DAH","ADH","BXW", "BQJ"}
62+
fmt.Println("Base64 to flag : ",FT.Urlenc_to_flag(FT.CreateUrlEncodedflag(res)))
63+
}*/
64+
65+
/*
66+
start := time.Now()
67+
trie.CheckDN1()
68+
elapsed := time.Since(start)
69+
fmt.Printf("Time Diff %s\n",elapsed)*/
70+
f, err := os.Create("./amx")
71+
if err != nil {
72+
fmt.Println("could not create memory profile: ", err)
73+
}
74+
defer f.Close() // error handling omitted for example
75+
if err := pprof.WriteHeapProfile(f); err != nil {
76+
fmt.Println("could not write memory profile: ", err)
77+
}
78+
79+
}
80+
81+
func PrintMemUsage() {
82+
var m runtime.MemStats
83+
runtime.ReadMemStats(&m)
84+
// For info on each, see: https://golang.org/pkg/runtime/#MemStats
85+
fmt.Printf("Alloc = %v MiB", bToMb(m.Alloc))
86+
fmt.Printf("\tTotalAlloc = %v MiB", bToMb(m.TotalAlloc))
87+
fmt.Printf("\tSys = %v MiB", bToMb(m.Sys))
88+
fmt.Printf("\tNumGC = %v\n", m.NumGC)
89+
}
90+
91+
func bToMb(b uint64) uint64 {
92+
return b / 1024 / 1024
5093
}

‎trie/commons.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ func Find_Lista_Listb(list1 []string, list2 []string) (bool, []string) {
114114
}
115115
}
116116

117-
fmt.Println("usr list : ", list1)
118-
fmt.Println("list 2 : ", list2)
117+
// fmt.Println("usr list : ", list1)
118+
// fmt.Println("list 2 : ", list2)
119119
return found, retlist
120120
}

‎trie/frozentrie.go‎

Lines changed: 137 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"io/ioutil"
99
"net/url"
1010
"strings"
11+
"encoding/binary"
1112
)
1213

1314
type FrozenTrie struct {
@@ -32,6 +33,14 @@ type FrozenTrie struct {
3233
usr_bl []string
3334
}
3435

36+
func (f *FrozenTrie) GetData() BS {
37+
return f.data
38+
}
39+
40+
func (f *FrozenTrie) GetDir() RankDirectory {
41+
return f.directory
42+
}
43+
3544
func (FT *FrozenTrie) Init(trieData []uint16, rdir RankDirectory, nodeCount int) {
3645
FT.data = BS{}
3746
FT.data.Init(trieData)
@@ -62,17 +71,17 @@ func (FT *FrozenTrie) Init(trieData []uint16, rdir RankDirectory, nodeCount int)
6271
FT.usr_bl = []string{}
6372
}
6473

65-
func (FT FrozenTrie) getNodeByIndex(index int) FrozenTrieNode {
74+
func (FT *FrozenTrie) getNodeByIndex(index int) FrozenTrieNode {
6675
FTN := FrozenTrieNode{}
6776
FTN.Init(FT, index)
6877
return FTN
6978
}
7079

71-
func (FT FrozenTrie) getRoot() FrozenTrieNode {
80+
func (FT *FrozenTrie) getRoot() FrozenTrieNode {
7281
return FT.getNodeByIndex(0)
7382
}
7483

75-
func (FT FrozenTrie) lookup(word []uint8) (bool, []uint32) {
84+
func (FT *FrozenTrie) lookup(word []uint8) (bool, []uint32) {
7685
var node = FT.getRoot()
7786
var emptyreturn []uint32
7887

@@ -260,7 +269,7 @@ func (FT *FrozenTrie) LoadTag() error {
260269
return nil
261270
}
262271

263-
func (FT FrozenTrie) FlagstoTag(flags []uint32) []string {
272+
func (FT *FrozenTrie) FlagstoTag(flags []uint32) []string {
264273
// flags has to be an array of 16-bit integers.
265274
header := uint16(flags[0])
266275
tagIndices := []int{}
@@ -309,8 +318,20 @@ func (FT *FrozenTrie) DNlookup(dn string, usr_flag string) (bool, []string) {
309318

310319
if FT.usr_flag == "" || FT.usr_flag != usr_flag {
311320
//fmt.Println("User config saved : ")
321+
var blocklists []string
322+
var err error
323+
s := strings.Split(usr_flag, ":")
324+
if len(s) > 1 {
325+
blocklists, err = FT.decode(s[1], s[0])
326+
} else {
327+
blocklists, err = FT.decode(usr_flag, "0")
328+
}
329+
if (err != nil) {
330+
fmt.Println(err, s)
331+
return false, nil
332+
}
333+
FT.usr_bl = blocklists
312334
FT.usr_flag = usr_flag
313-
FT.usr_bl = FT.Urlenc_to_flag(FT.usr_flag)
314335
}
315336

316337
dn = strings.TrimSpace(dn)
@@ -362,7 +383,7 @@ func (FT *FrozenTrie) DNlookup(dn string, usr_flag string) (bool, []string) {
362383
}
363384
}
364385

365-
func (FT FrozenTrie) CreateUrlEncodedflag(fl []string) string {
386+
func (FT *FrozenTrie) CreateUrlEncodedflag(fl []string) string {
366387
var val = 0
367388
var res = ""
368389
for _, flag := range fl {
@@ -418,9 +439,118 @@ func (FT FrozenTrie) CreateUrlEncodedflag(fl []string) string {
418439
return url.QueryEscape(b64.StdEncoding.EncodeToString([]byte(res)))
419440
}
420441

421-
func (FT FrozenTrie) Urlenc_to_flag(str string) []string {
442+
func (FT *FrozenTrie) Urlenc_to_flag(str string) []string {
422443
str, _ = url.QueryUnescape(str)
423444
buf, _ := b64.StdEncoding.DecodeString(str)
424445
str = string(buf)
425446
return FT.FlagstoTag(Flag_to_uint(str))
426447
}
448+
449+
func (FT *FrozenTrie) decode(stamp string, ver string) (tags []string, err error) {
450+
decoder := b64.StdEncoding
451+
if (ver == "0") {
452+
stamp, err = url.QueryUnescape(stamp)
453+
} else if (ver == "1") {
454+
stamp, err = url.PathUnescape(stamp)
455+
decoder = b64.URLEncoding
456+
} else {
457+
err = fmt.Errorf("version does not exist", ver)
458+
}
459+
460+
if err != nil {
461+
return nil, err
462+
}
463+
464+
buf, err := decoder.DecodeString(stamp)
465+
if err != nil {
466+
fmt.Println("b64", stamp)
467+
return
468+
}
469+
470+
var u16 []uint16
471+
if ver == "0" {
472+
u16 = stringtouint(string(buf))
473+
} else if ver == "1" {
474+
u16 = bytestouint(buf)
475+
}
476+
fmt.Println("%s %v", ver, u16)
477+
return FT.flagstotag(u16)
478+
}
479+
480+
func (ft *FrozenTrie) flagstotag(flags []uint16) ([]string, error) {
481+
// flags has to be an array of 16-bit integers.
482+
483+
// first index always contains the header
484+
header := uint16(flags[0])
485+
// store of each big-endian position of set bits in header
486+
tagIndices := []int{}
487+
values := []string{}
488+
var mask uint16
489+
490+
// b1000,0000,0000,0000
491+
mask = 0x8000
492+
493+
// read first 16 header bits from msb to lsb
494+
// and capture indices of set bits in tagIndices
495+
for i := 0; i < 16; i++ {
496+
if (header << i) == 0 {
497+
break
498+
}
499+
if (header & mask) == mask {
500+
tagIndices = append(tagIndices, i)
501+
}
502+
mask = mask >> 1 // shift to read the next msb bit
503+
}
504+
// the number of set bits in header must correspond to total
505+
// blocklist "flags" excluding the header at position 0
506+
if len(tagIndices) != (len(flags) - 1) {
507+
err := fmt.Errorf("%v %v flags and header mismatch", tagIndices, flags)
508+
return nil, err
509+
}
510+
511+
// for all blocklist flags excluding the header
512+
// figure out the blocklist-ids
513+
for i := 1; i < len(flags); i++ {
514+
// 16 blocklists are represented by one flag
515+
// that is, one bit per blocklist
516+
var flag = uint16(flags[i])
517+
// get the index of the current flag in the header
518+
var index = tagIndices[i-1]
519+
mask = 0x8000
520+
// for each of the 16 bits in the flag
521+
// capture the set bits and calculate
522+
// its actual decimal value, the blocklist-id
523+
for j := 0; j < 16; j++ {
524+
if (flag << j) == 0 {
525+
break
526+
}
527+
if (flag & mask) == mask {
528+
pos := (index * 16) + j
529+
// from the decimal value which is its
530+
// blocklist-id, fetch its metadata
531+
values = append(values, FT.rflags[pos])
532+
}
533+
mask = mask >> 1
534+
}
535+
}
536+
return values, nil
537+
}
538+
539+
func stringtouint(str string) []uint16 {
540+
runedata := []rune(str)
541+
resp := make([]uint16, len(runedata))
542+
for key, value := range runedata {
543+
resp[key] = uint16(value)
544+
}
545+
return resp
546+
}
547+
548+
func bytestouint(b []byte) []uint16 {
549+
data := make([]uint16, len(b)/2)
550+
for i := range data {
551+
// assuming little endian
552+
data[i] = binary.LittleEndian.Uint16(b[i*2 : (i+1)*2])
553+
}
554+
return data
555+
}
556+

‎trie/frozentrienode.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,15 @@ package trie
33
import "fmt"
44

55
type FrozenTrieNode struct {
6-
trie FrozenTrie
6+
trie *FrozenTrie
77
index int
88
valCached *[]uint32
99
finCached, comCached, flagCached *bool
1010
whCached *uint32
1111
fcCached, chCached *int
1212
}
1313

14-
func (FTN *FrozenTrieNode) Init(FT FrozenTrie, index int) {
14+
func (FTN *FrozenTrieNode) Init(FT *FrozenTrie, index int) {
1515
FTN.trie = FT
1616
FTN.index = index
1717
if Debug {

0 commit comments

Comments
 (0)