Merge pull request #14 from halprin/filter-delete

Filter Expressions for Delete

Author: halprin

Makefile

@@ -3,6 +3,7 @@ compile:
 
 runTestDynamoDB:
 	docker-compose -f dynamodb-docker-compose.yml up -d
+	echo "Navigate to http://localhost:8001 for DynamoDB-Admin"
 
 loadTestData:
 	./generate_mass_data.sh 500

README.md

@@ -12,12 +12,25 @@ _**Warning**: running this command will result in all the items in the specified
 is no "are you sure?" prompt._
 
 ```shell
-delete-dynamodb-items <table name> [--endpoint=URL]
+delete-dynamodb-items <table name> [--endpoint=URL] [--filter-expression=string] [--expression-attribute-names=JSON] [--expression-attribute-values=JSON]
 ```
 
 The program uses the default AWS credential algorithm to determine what IAM entity and region is used.  E.g. the
 `~/.aws/credentials` file, the `AWS_*` environment variables, etc.
 
+## Filter Expressions
+
+You can specify a special expression to filter out items you don't want deleted.  AKA, the item will be deleted if the
+filter matches.  You can learn more about filter expressions in
+[AWS's DynamoDB Developer Guide](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Scan.html#Scan.FilterExpression)
+and the
+[`filter-expression` section in the AWS CLI](https://docs.aws.amazon.com/cli/latest/reference/dynamodb/scan.html).
+
+Use a combination of the `--filter-expression=`, `--expression-attribute-names=`, and `--expression-attribute-values=`
+options.  These options work the same way as the options on the AWS CLI.
+
+E.g. `--filter-expression='#k > :v' --expression-attribute-names='{"#k": "number"}' --expression-attribute-values='{":v": {"N": "50"}}'`
+
 ### Custom Endpoint
 
 You can customize the DynamoDB endpoint with the `--endpoint=` (or `-e`) option.  Set it to the URL of the endpoint.
@@ -28,6 +41,5 @@ E.g. `--endpoint=http://localhost:8002`.  If unspecified, the default AWS endpoi
 Run the following to compile your own copy from source.
 
 ```shell
-go get -v -t -d ./cmd/
 go build -o delete-dynamodb-items -v ./cmd/
 ```

config/config.go

@@ -2,6 +2,9 @@ package config
 
 var tableName *string
 var dynamoDbEndpoint *string
+var filterExpression *string
+var expressionAttributeNames *string
+var expressionAttributeValues *string
 
 func SetTableName(name string) {
 	tableName = &name
@@ -18,3 +21,27 @@ func SetDynamoDbEndpoint(endpoint string) {
 func GetDynamoDbEndpoint() *string {
 	return dynamoDbEndpoint
 }
+
+func SetFilterExpression(expression string) {
+	filterExpression = &expression
+}
+
+func GetFilterExpression() *string {
+	return filterExpression
+}
+
+func SetExpressionAttributeNames(names string) {
+	expressionAttributeNames = &names
+}
+
+func GetExpressionAttributeNames() *string {
+	return expressionAttributeNames
+}
+
+func SetExpressionAttributeValues(values string) {
+	expressionAttributeValues = &values
+}
+
+func GetExpressionAttributeValues() *string {
+	return expressionAttributeValues
+}

dynamo/dynamo.go

@@ -36,7 +36,11 @@ func DeleteAllItemsInTable() error {
 	goroutinePool := parallel.NewPool(concurrency, 41944)
 	defer goroutinePool.Release()
 
-	for subItemList := range getItemsGoroutine(tableName) {
+	expressionFilter := config.GetFilterExpression()
+	expressionAttributeNames := config.GetExpressionAttributeNames()
+	expressionAttributeValues := config.GetExpressionAttributeValues()
+
+	for subItemList := range getItemsGoroutine(tableName, expressionFilter, expressionAttributeNames, expressionAttributeValues) {
 		err = deleteItems(subItemList, tableName, goroutinePool)
 		if err != nil {
 			return err

dynamo/scan.go

@@ -1,32 +1,57 @@
 package dynamo
 
 import (
+	"encoding/json"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/dynamodb"
 	"log"
 )
 
-func getItemsGoroutine(tableName string) chan []map[string]*dynamodb.AttributeValue {
+type expressionAttributeNamesType map[string]*string
+type expressionAttributeValuesType map[string]*dynamodb.AttributeValue
+
+func getItemsGoroutine(tableName string, filterExpression *string, expressionAttributeNames *string, expressionAttributeValues *string) chan []map[string]*dynamodb.AttributeValue {
 	yield := make(chan []map[string]*dynamodb.AttributeValue)
 
+	var names expressionAttributeNamesType
+	if expressionAttributeNames != nil {
+		err := json.Unmarshal([]byte(*expressionAttributeNames), &names)
+		if err != nil {
+			log.Printf("Failed to unmarshal the expression attribute names, %+v", err)
+			return nil
+		}
+	}
+
+	var values expressionAttributeValuesType
+	if expressionAttributeValues != nil {
+		err := json.Unmarshal([]byte(*expressionAttributeValues), &values)
+		if err != nil {
+			log.Printf("Failed to unmarshal the expression attribute values, %+v", err)
+			return nil
+		}
+	}
+
 	go func() {
 		scanInput := &dynamodb.ScanInput{
-			TableName: aws.String(tableName),
+			TableName:                aws.String(tableName),
+			FilterExpression:         filterExpression,
+			ExpressionAttributeNames:  names,
+			ExpressionAttributeValues: values,
 		}
 
 		for {
 			log.Println("Scanning items")
 
 			scanOutput, err := dynamoService.Scan(scanInput)
 			if err != nil {
-				log.Println("Failed to scan the items")
+				log.Printf("Failed to scan the items, %+v", err)
 				break
 			}
 
 			yield <- scanOutput.Items
 
 			if scanOutput.LastEvaluatedKey != nil && len(scanOutput.LastEvaluatedKey) > 0 {
-				//there are still items to scan, the the key to start scanning from again
+				//there are still items to scan, set the key to start scanning from again
 				scanInput.ExclusiveStartKey = scanOutput.LastEvaluatedKey
 			} else {
 				//no more items to scan, break out

external/cli/cli.go

@@ -3,6 +3,7 @@ package cli
 import (
 	"github.com/halprin/delete-dynamodb-items/config"
 	"github.com/teris-io/cli"
+	"log"
 	"os"
 )
 
@@ -11,12 +12,25 @@ func FillConfig() {
 	tableNameCliArg := cli.NewArg("table name", "The name of the table for which all the items will be deleted").WithType(cli.TypeString)
 	endpointCliOption := cli.NewOption(endpointKey, "A URL of the DynamoDB endpoint to use").WithChar('e').WithType(cli.TypeString)
 
-	parser := cli.New("Deletes all the items in a DynamoDB table").WithArg(tableNameCliArg).WithOption(endpointCliOption)
+	filterExpressionKey := "filter-expression"
+	expressionAttributeNamesKey := "expression-attribute-names"
+	expressionAttributeValuesKey := "expression-attribute-values"
+	filterExpressionOption := cli.NewOption(filterExpressionKey, "A filter expression determines which items within the Scan results should be returned to you.  All of the other results are discarded.").WithType(cli.TypeString)
+	expressionAttributeNamesOption := cli.NewOption(expressionAttributeNamesKey, "Way to specify names in the filter expression that are DynamoDB reserved words.").WithType(cli.TypeString)
+	expressionAttributeValuesOption := cli.NewOption(expressionAttributeValuesKey, "Way to specify values in the filter expression that are DynamoDB reserved words.").WithType(cli.TypeString)
+
+	parser := cli.New("Deletes all the items in a DynamoDB table").
+		WithArg(tableNameCliArg).
+		WithOption(endpointCliOption).
+		WithOption(filterExpressionOption).
+		WithOption(expressionAttributeNamesOption).
+		WithOption(expressionAttributeValuesOption)
 
 	invocation, arguments, options, err := parser.Parse(os.Args)
 	help, helpExistsInOptions := options["help"]
 
 	if err != nil {
+		log.Printf("Error: %+v", err)
 		_ = parser.Usage(invocation, os.Stdout)
 		os.Exit(1)
 	} else if helpExistsInOptions && help == "true" {
@@ -31,4 +45,19 @@ func FillConfig() {
 	if endpointExistsInOptions {
 		config.SetDynamoDbEndpoint(endpoint)
 	}
+
+	expression, expressionExistsInOptions := options[filterExpressionKey]
+	if expressionExistsInOptions {
+		config.SetFilterExpression(expression)
+	}
+
+	names, namesExistsInOptions := options[expressionAttributeNamesKey]
+	if namesExistsInOptions {
+		config.SetExpressionAttributeNames(names)
+	}
+
+	values, valuesExistsInOptions := options[expressionAttributeValuesKey]
+	if valuesExistsInOptions {
+		config.SetExpressionAttributeValues(values)
+	}
 }

generate_mass_data.sh

@@ -12,7 +12,8 @@ items_ending=']}'
 lorem_ipsum='Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a efficitur nunc. Morbi fermentum sem metus, vel venenatis leo porttitor quis. Etiam maximus neque a pharetra viverra. Sed turpis lacus, blandit ac tortor elementum, scelerisque feugiat risus. Nam malesuada augue et purus aliquet, et semper dolor cursus. Suspendisse volutpat dolor nec efficitur rutrum. Aliquam leo libero, posuere eget vulputate in, luctus nec nibh. Donec eu tellus eu libero scelerisque molestie. Ut sed pretium nibh. Donec suscipit eget dui quis lacinia. Aliquam non pulvinar massa, nec blandit lectus. Cras sollicitudin rhoncus ex. Nunc ipsum dui, dictum in risus nec, convallis rutrum justo. In tempor dui nisl, in fringilla massa vehicula ac. Donec a ipsum luctus, venenatis magna ut, venenatis risus. Vivamus eu dapibus odio. Aenean dapibus urna orci, sed pharetra nunc dapibus ac. Praesent ornare, felis sit amet mattis faucibus, odio arcu laoreet arcu, eu blandit nisi turpis cursus enim.'
 
 for ((index = 1 ; index <= num_items ; index++)); do
-    current_request="{\"PutRequest\": {\"Item\": {\"id\": {\"S\": \"$(uuidgen)\"}, \"text\": {\"S\": \"${lorem_ipsum}\"}}}}"
+    rand_number=$((RANDOM % 100))  # not actually random, but lazy with the modulus
+    current_request="{\"PutRequest\": {\"Item\": {\"id\": {\"S\": \"$(uuidgen)\"}, \"text\": {\"S\": \"${lorem_ipsum}\"}, \"number\": {\"N\": \"${rand_number}\"}}}}"
     items_middle="${items_middle}${current_request},"
     if [[ $((index % 25)) == 0 ]]; then
         items_middle=${items_middle::${#items_middle}-1}