diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..b3a27a3 --- /dev/null +++ b/.env.example @@ -0,0 +1,6 @@ +ARCHIVER_ADDR= +S3_ENDPOINT= +S3_ACCESS= +S3_SECRET= +S3_BUCKET= +PRODUCTION_MODE= \ No newline at end of file diff --git a/cmd/cleanuser/main.go b/cmd/cleanuser/main.go index bffdf6a..8c12f18 100644 --- a/cmd/cleanuser/main.go +++ b/cmd/cleanuser/main.go @@ -10,15 +10,11 @@ import ( "strconv" "strings" - "github.com/TicketsBot-cloud/gdl/objects/channel/message" "github.com/TicketsBot-cloud/logarchiver/pkg/config" - "github.com/TicketsBot-cloud/logarchiver/pkg/model" - v1 "github.com/TicketsBot-cloud/logarchiver/pkg/model/v1" v2 "github.com/TicketsBot-cloud/logarchiver/pkg/model/v2" "github.com/TicketsBot-cloud/logarchiver/pkg/s3client" + "github.com/TicketsBot-cloud/logarchiver/pkg/utils" "github.com/TicketsBot/common/encryption" - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" ) var ( @@ -39,17 +35,11 @@ func main() { panic("either -ticket, -all or -csv must be set") } - client, err := minio.New(cfg.Endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""), - Secure: cfg.Secure, - }) - + s3Client, err := s3client.NewFromCliConfig(cfg) if err != nil { panic(err) } - s3Client := s3client.NewS3Client(client, cfg.Bucket) - var count int if *csv != "" { tickets := parseCsv(*csv) @@ -77,7 +67,7 @@ func main() { } for _, key := range keys { - ticketId, err := strconv.Atoi(key[strings.LastIndex(key, "/")+1:]) + ticketId, err := s3client.TicketIDFromKey(key) if err != nil { fmt.Printf("error occurred while parsing id of %s: %v\n", key, err) continue @@ -130,23 +120,9 @@ func clean(client *s3client.S3Client, guildId uint64, ticketId int) (int, error) panic(err) } - var transcript v2.Transcript - - version := model.GetVersion(data) - switch version { - case model.V1: - var messages []message.Message - if err := json.Unmarshal(data, &messages); err != nil { - panic(err) - } - - transcript = v1.ConvertToV2(messages) - case model.V2: - if err := json.Unmarshal(data, &transcript); err != nil { - panic(err) - } - default: - panic(fmt.Sprintf("Unknown version %d", version)) + transcript, err := utils.Decode(data) + if err != nil { + return 0, err } transcript.Entities.Users[*userId] = v2.User{ @@ -204,9 +180,10 @@ func parseCsv(file string) map[uint64][]int { ticketIdIdx := -1 for i, h := range header { - if h == "guild_id" { + switch h { + case "guild_id": guildIdIdx = i - } else if h == "ticket_id" || h == "id" { + case "ticket_id", "id": ticketIdIdx = i } } diff --git a/cmd/deletetranscript/main.go b/cmd/deletetranscript/main.go index b452f67..8987ff6 100644 --- a/cmd/deletetranscript/main.go +++ b/cmd/deletetranscript/main.go @@ -9,8 +9,6 @@ import ( "github.com/TicketsBot-cloud/logarchiver/pkg/config" "github.com/TicketsBot-cloud/logarchiver/pkg/s3client" - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" _ "github.com/joho/godotenv/autoload" ) @@ -29,17 +27,11 @@ func main() { panic("guild id must be set") } - m, err := minio.New(cfg.Endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""), - Secure: cfg.Secure, - }) - + client, err := s3client.NewFromCliConfig(cfg) if err != nil { panic(err) } - client := s3client.NewS3Client(m, cfg.Bucket) - if *all { keys, err := client.GetAllKeysForGuild(context.Background(), *guildId) if err != nil { @@ -47,7 +39,7 @@ func main() { } for _, key := range keys { - ticketId, err := strconv.Atoi(key[strings.LastIndex(key, "/")+1:]) + ticketId, err := s3client.TicketIDFromKey(key) if err != nil { fmt.Printf("error occurred while parsing id of %s: %v\n", key, err) continue diff --git a/cmd/exportguild/main.go b/cmd/exportguild/main.go index d1e9982..24ca0fb 100644 --- a/cmd/exportguild/main.go +++ b/cmd/exportguild/main.go @@ -7,18 +7,12 @@ import ( "flag" "fmt" "os" - "strconv" - "strings" - "github.com/TicketsBot-cloud/gdl/objects/channel/message" "github.com/TicketsBot-cloud/logarchiver/pkg/config" - "github.com/TicketsBot-cloud/logarchiver/pkg/model" - v1 "github.com/TicketsBot-cloud/logarchiver/pkg/model/v1" v22 "github.com/TicketsBot-cloud/logarchiver/pkg/model/v2" "github.com/TicketsBot-cloud/logarchiver/pkg/s3client" + "github.com/TicketsBot-cloud/logarchiver/pkg/utils" "github.com/TicketsBot/common/encryption" - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" "golang.org/x/sync/errgroup" ) @@ -37,17 +31,11 @@ func main() { flag.Parse() conf := config.Parse[config.CliConfig]() - // create minio client - m, err := minio.New(conf.Endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(conf.AccessKey, conf.SecretKey, ""), - Secure: conf.Secure, - }) + client, err := s3client.NewFromCliConfig(conf) if err != nil { panic(err) } - client := s3client.NewS3Client(m, conf.Bucket) - // likely to be file exists _ = os.Mkdir(fmt.Sprintf("export/%d", *guildId), 0) @@ -69,11 +57,10 @@ func main() { }() group, _ := errgroup.WithContext(context.Background()) - for i := 0; i < workers; i++ { + for range workers { group.Go(func() error { for key := range keyCh { - id := key[strings.LastIndex(key, "/")+1:] - parsed, err := strconv.Atoi(id) + parsed, err := s3client.TicketIDFromKey(key) must(err) if after != nil && *after > 0 && parsed < *after { @@ -104,24 +91,8 @@ func export(id int, client *s3client.S3Client) { must(err) if *convert || (userWhitelist != nil && *userWhitelist > 0) { - var transcript v22.Transcript - - version := model.GetVersion(data) - switch version { - case model.V1: - var messages []message.Message - if err := json.Unmarshal(data, &messages); err != nil { - panic(err) - } - - transcript = v1.ConvertToV2(messages) - case model.V2: - if err := json.Unmarshal(data, &transcript); err != nil { - panic(err) - } - default: - panic(fmt.Sprintf("Unknown version %d", version)) - } + transcript, err := utils.Decode(data) + must(err) data, err = json.Marshal(transcript) must(err) diff --git a/cmd/exportuser/cachedata.go b/cmd/exportuser/cachedata.go deleted file mode 100644 index 7f3c041..0000000 --- a/cmd/exportuser/cachedata.go +++ /dev/null @@ -1,37 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "strconv" - - "github.com/TicketsBot-cloud/gdl/cache" -) - -func getCacheData(cache *cache.PgCache, userId uint64) map[string]interface{} { - data := make(map[string]interface{}) - - user, ok := cache.GetUser(userId) - if ok { - data["user"] = user - } else { - data["user"] = nil - } - - rows, err := cache.Query(context.Background(), `SELECT guild_id, data FROM members WHERE "user_id" = $1;`, userId) - must(err) - - memberData := make(map[string]interface{}) - for rows.Next() { - var guildId uint64 - var raw string - - must(rows.Scan(&guildId, &raw)) - - memberData[strconv.FormatUint(guildId, 10)] = json.RawMessage([]byte(raw)) - } - - data["member_data"] = memberData - - return data -} diff --git a/cmd/exportuser/main.go b/cmd/exportuser/main.go index 1f74977..08fdeb9 100644 --- a/cmd/exportuser/main.go +++ b/cmd/exportuser/main.go @@ -11,17 +11,13 @@ import ( "time" "github.com/TicketsBot-cloud/gdl/cache" - "github.com/TicketsBot-cloud/gdl/objects/channel/message" "github.com/TicketsBot-cloud/logarchiver/pkg/config" - "github.com/TicketsBot-cloud/logarchiver/pkg/model" - v1 "github.com/TicketsBot-cloud/logarchiver/pkg/model/v1" v2 "github.com/TicketsBot-cloud/logarchiver/pkg/model/v2" "github.com/TicketsBot-cloud/logarchiver/pkg/s3client" + "github.com/TicketsBot-cloud/logarchiver/pkg/utils" "github.com/TicketsBot/common/encryption" "github.com/TicketsBot/database" "github.com/jackc/pgx/v4/pgxpool" - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" "go.uber.org/zap" ) @@ -34,7 +30,7 @@ var ( func main() { flag.Parse() - conf := config.Parse() + conf := config.Parse[config.CliConfig]() // likely to be file exists _ = os.Mkdir(fmt.Sprintf("export_user/%d", *userId), 0) @@ -125,18 +121,12 @@ func main() { getTranscripts(conf, transcriptIds) } -func getTranscripts(conf config.Config, tickets map[uint64][]int) { - // create minio client - m, err := minio.New(conf.Endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(conf.AccessKey, conf.SecretKey, ""), - Secure: conf.Secure, - }) +func getTranscripts(conf config.CliConfig, tickets map[uint64][]int) { + client, err := s3client.NewFromCliConfig(conf) if err != nil { panic(err) } - client := s3client.NewS3Client(m, conf.Bucket) - logger, err := zap.NewDevelopment() if err != nil { panic(err) @@ -168,24 +158,10 @@ func getTranscripts(conf config.Config, tickets map[uint64][]int) { continue } - // Convert to v2 if needed - var transcript v2.Transcript - - version := model.GetVersion(data) - switch version { - case model.V1: - var messages []message.Message - if err := json.Unmarshal(data, &messages); err != nil { - panic(err) - } - - transcript = v1.ConvertToV2(messages) - case model.V2: - if err := json.Unmarshal(data, &transcript); err != nil { - panic(err) - } - default: - panic(fmt.Sprintf("Unknown version %d", version)) + transcript, err := utils.Decode(data) + if err != nil { + logger.Error("failed to decode transcript", zap.Error(err), zap.Uint64("guildId", guildId), zap.Int("ticketId", ticketId)) + continue } transcript.Entities.Channels = nil @@ -486,8 +462,8 @@ WHERE "activated_by" = $1;` func getCacheData(cache *cache.PgCache, userId uint64) map[string]interface{} { data := make(map[string]interface{}) - user, ok := cache.GetUser(userId) - if ok { + user, err := cache.GetUser(context.Background(), userId) + if err == nil { data["user"] = user } else { data["user"] = nil diff --git a/cmd/exportuser/userdata.go b/cmd/exportuser/userdata.go deleted file mode 100644 index c9ecbf5..0000000 --- a/cmd/exportuser/userdata.go +++ /dev/null @@ -1,2 +0,0 @@ -package main - diff --git a/cmd/fix-file-names/main.go b/cmd/fix-file-names/main.go index e6cb4dd..7abb497 100644 --- a/cmd/fix-file-names/main.go +++ b/cmd/fix-file-names/main.go @@ -2,7 +2,6 @@ package main import ( "context" - "flag" "strings" "sync" "sync/atomic" @@ -16,8 +15,7 @@ import ( const workers = 30 func main() { - flag.Parse() - conf := config.Parse() + conf := config.Parse[config.CliConfig]() // create minio client client, err := minio.New(conf.Endpoint, &minio.Options{ diff --git a/envvars.md b/envvars.md deleted file mode 100644 index 9fd88fd..0000000 --- a/envvars.md +++ /dev/null @@ -1,6 +0,0 @@ -- ARCHIVER_ADDR -- S3_ENDPOINT -- S3_ACCESS -- S3_SECRET -- S3_BUCKET -- PRODUCTION_MODE \ No newline at end of file diff --git a/go.mod b/go.mod index e5656e6..41eaf69 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22.0 toolchain go1.24.2 -// replace github.com/TicketsBot-cloud/gdl => ../gdl +replace github.com/TicketsBot-cloud/gdl => ../gdl require ( github.com/TicketsBot-cloud/gdl v0.0.0-20250509054940-2045fbe19c06 @@ -18,6 +18,7 @@ require ( github.com/jackc/pgx/v4 v4.18.3 github.com/jackc/pgx/v5 v5.6.0 github.com/joho/godotenv v1.5.1 + github.com/klauspost/compress v1.17.9 github.com/minio/minio-go/v7 v7.0.73 go.uber.org/zap v1.23.0 golang.org/x/sync v0.8.0 @@ -44,7 +45,6 @@ require ( github.com/jackc/puddle v1.3.0 // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/mattn/go-isatty v0.0.17 // indirect diff --git a/go.sum b/go.sum index b79acb0..1f657c1 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,5 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/TicketsBot-cloud/gdl v0.0.0-20250509054940-2045fbe19c06 h1:PzziB2S58d9agJtpaPVrYMTuBiJICr2QIGQoqL6l3z0= -github.com/TicketsBot-cloud/gdl v0.0.0-20250509054940-2045fbe19c06/go.mod h1:CdwBR2egPtxUXjD2CgC9ZwfuB8dz9HPePM8nuG6dt7Y= github.com/TicketsBot/common v0.0.0-20230723121853-8d873b27086e h1:K7c0kRXxaW14dl9G9rQ78eGP+Lu+F7qPxTud/j68SQQ= github.com/TicketsBot/common v0.0.0-20230723121853-8d873b27086e/go.mod h1:rZuaTbjajlxscl628aBgUGiUZcOVLvF6pfsPL+2JOac= github.com/TicketsBot/database v0.0.0-20220217133004-d190910ad66f h1:kBIHaAxyIGqxgwz/ZRggNGjyIdZ3ctWZqjJ9Svrn4L4= diff --git a/pkg/config/config.go b/pkg/config/config.go index 7c59216..5b9fbb7 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -23,7 +23,7 @@ type Config struct { func Parse[T any]() (conf T) { parsers := env.CustomParsers{ - reflect.TypeOf(uuid.UUID{}): func(value string) (interface{}, error) { + reflect.TypeFor[uuid.UUID](): func(value string) (any, error) { return uuid.Parse(value) }, } diff --git a/pkg/model/v1/attachment.go b/pkg/model/v1/attachment.go index a7dda2b..6dcfa59 100644 --- a/pkg/model/v1/attachment.go +++ b/pkg/model/v1/attachment.go @@ -1,6 +1,6 @@ -package v1 - -type Attachment struct { - Filename string `json:"filename"` - Url string `json:"url"` -} +package v1 + +type Attachment struct { + Filename string `json:"filename"` + Url string `json:"url"` +} diff --git a/pkg/model/v2/entities.go b/pkg/model/v2/entities.go index 6f6e11b..9f4c73e 100644 --- a/pkg/model/v2/entities.go +++ b/pkg/model/v2/entities.go @@ -47,9 +47,13 @@ func UserFromGdl(entity user.User) User { } func ChannelFromGdl(entity channel.Channel) Channel { + name := "" + if entity.Name != nil { + name = *entity.Name + } return Channel{ Id: entity.Id, - Name: entity.Name, + Name: name, Type: entity.Type, } } diff --git a/pkg/model/version.go b/pkg/model/version.go index 836f494..9aae048 100644 --- a/pkg/model/version.go +++ b/pkg/model/version.go @@ -28,4 +28,4 @@ func GetVersion(payload []byte) Version { } return decoded.Version -} \ No newline at end of file +} diff --git a/pkg/s3client/client.go b/pkg/s3client/client.go index bbb4ce2..0494cbc 100644 --- a/pkg/s3client/client.go +++ b/pkg/s3client/client.go @@ -6,10 +6,14 @@ import ( "errors" "fmt" "io" + "strconv" + "strings" + "github.com/TicketsBot-cloud/logarchiver/pkg/config" "github.com/TicketsBot-cloud/logarchiver/pkg/repository/model" "github.com/klauspost/compress/zstd" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) type S3Client struct { @@ -25,6 +29,18 @@ func NewS3Client(client *minio.Client, bucketName string) *S3Client { } } +func NewFromCliConfig(cfg config.CliConfig) (*S3Client, error) { + m, err := minio.New(cfg.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""), + Secure: cfg.Secure, + }) + if err != nil { + return nil, err + } + + return NewS3Client(m, cfg.Bucket), nil +} + func (c *S3Client) GetTicket(ctx context.Context, guildId uint64, ticketId int) ([]byte, error) { key := fmt.Sprintf("%d/%d", guildId, ticketId) @@ -109,6 +125,11 @@ func (c *S3Client) BucketName() string { return c.bucketName } +// TicketIDFromKey extracts the ticket ID from an S3 object key of the form "guildId/ticketId". +func TicketIDFromKey(key string) (int, error) { + return strconv.Atoi(key[strings.LastIndex(key, "/")+1:]) +} + func isNotFoundErr(err error) bool { var resp minio.ErrorResponse return errors.As(err, &resp) && resp.Code == "NoSuchKey" diff --git a/pkg/utils/transcripts.go b/pkg/utils/transcripts.go new file mode 100644 index 0000000..6e3674e --- /dev/null +++ b/pkg/utils/transcripts.go @@ -0,0 +1,32 @@ +package utils + +import ( + "encoding/json" + "fmt" + + "github.com/TicketsBot-cloud/gdl/objects/channel/message" + "github.com/TicketsBot-cloud/logarchiver/pkg/model" + v1 "github.com/TicketsBot-cloud/logarchiver/pkg/model/v1" + v2 "github.com/TicketsBot-cloud/logarchiver/pkg/model/v2" +) + +// Decode parses a raw transcript payload into a v2.Transcript, converting from v1 if necessary. +func Decode(data []byte) (v2.Transcript, error) { + version := model.GetVersion(data) + switch version { + case model.V1: + var messages []message.Message + if err := json.Unmarshal(data, &messages); err != nil { + return v2.Transcript{}, err + } + return v1.ConvertToV2(messages), nil + case model.V2: + var t v2.Transcript + if err := json.Unmarshal(data, &t); err != nil { + return v2.Transcript{}, err + } + return t, nil + default: + return v2.Transcript{}, fmt.Errorf("unknown transcript version %d", version) + } +}