diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml new file mode 100644 index 0000000..e02c762 --- /dev/null +++ b/.github/workflows/pr-checks.yml @@ -0,0 +1,49 @@ +name: PR Checks + +on: + pull_request: + branches: [ main ] + +jobs: + format-check: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Black + run: | + python -m pip install --upgrade pip + pip install black + + - name: Check code formatting with Black + run: | + black --check *.py + + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.9, "3.10", "3.11", "3.12", "3.13"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run tests + run: | + pytest test_msstats.py -v \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3a223af..bbcd978 100644 --- a/.gitignore +++ b/.gitignore @@ -135,4 +135,7 @@ config.ini # Output files *.xlsx *.csv -*.json \ No newline at end of file +*.json + +# Claude +.claude \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..dbbbb0f --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,96 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +MSStats is a Python tool for extracting Google Cloud MemoryStore (Redis) database metrics using the Google Cloud Monitoring API. It processes Redis databases (single instance and replicated) across multiple GCP service accounts and generates Excel reports with usage statistics. + +## Architecture + +- **Single Python script**: `msstats.py` - Main application that processes Redis metrics +- **Batch processing**: Shell scripts for bulk operations across multiple GCP projects +- **Service account management**: Scripts to grant/revoke monitoring permissions +- **Output format**: Excel files with detailed Redis command statistics and throughput data + +## Key Components + +- **Metric Processing**: Categorizes Redis commands by type (Get, Set, Hash, List, etc.) from monitoring data +- **Multi-node handling**: Aggregates metrics across Redis cluster nodes taking maximum values +- **Time series data**: Configurable duration (default 7 days) and step intervals (default 60s) +- **GCP Integration**: Uses Google Cloud Monitoring API with service account authentication + +## Development Commands + +### Setup Environment +```bash +# Create and activate virtual environment +python -m venv .env && source .env/bin/activate + +# Install dependencies +pip install -r requirements.txt + +# Place service account JSON files in root directory +cp path/to/service_account.json . +``` + +### Running the Tool +```bash +# Basic usage (7 days, 60s steps) +python msstats.py + +# Custom duration and step size +python msstats.py --duration 1800 --step 300 + +# Specific project +python msstats.py -p project-id +``` + +### Batch Operations +```bash +# Grant monitoring permissions to service account +./grant_sa_monitoring_viewer.sh service-account@project.iam.gserviceaccount.com + +# Run across all accessible projects +./batch_run_msstats.sh + +# Remove monitoring permissions +./remove_sa_monitoring_viewer.sh service-account@project.iam.gserviceaccount.com + +# Clean up environment +deactivate +``` + +## Command Line Options + +- `--duration SECONDS`: Time period to analyze (default: 604800 = 7 days) +- `--step SECONDS`: Metric sampling interval (default: 60 seconds) +- `-p PROJECT_ID`: Target specific GCP project + +### Testing and Code Quality +```bash +# Run all tests (unit and integration) +pytest test_msstats.py + +# Format code with Black +black *.py + +# Check formatting without making changes +black --check *.py +``` + +## Dependencies + +- `openpyxl>=3.0.4`: Excel file generation +- `google-cloud-monitoring==2.18.0`: GCP Monitoring API client +- `black>=25.1.0`: Code formatter +- `pytest>=8.4.0`: Testing framework +- `pytest-mock>=3.14.0`: Mock utilities for testing +- Python 3.9+ required + +## Security Notes + +- Service account JSON files must be placed in repository root +- Tool uses read-only monitoring API access +- Never connects directly to Redis instances +- No impact on database performance or data \ No newline at end of file diff --git a/CODEOWNERS b/CODEOWNERS index e193874..a5fb2e2 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,7 +1,7 @@ # These owners will be the default owners for everything in # the repo. Unless a later match takes precedence will be requested # for review when someone opens a pull request. -* @thomasefthymiou78 @erniavr @mar1boroman @ewpreston @gmflau +* @ajGingrich @kurtfm @sagile @thomasefthymiou78 @mar1boroman # Order is important; the last matching pattern takes the most # precedence. When someone opens a pull request that only diff --git a/README.md b/README.md index 0ba5ce2..f95fbb6 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # MSSTATS -MSStats is a tool for extracting MemoryStore database metrics. The script is able to process all the Redis databases, both single instance and replicated (Basic or Standard) ones that belong to a specific service account. Multiple service accounts can be used at once. +MSStats is a tool for extracting MemoryStore database metrics. The script is able to process all the Redis databases, both single instance and replicated (Basic or Standard) ones that belong to a specific service account. Multiple service accounts can be used at once. The script will purely use google cloud monitoring api for getting the metrics. It will never connect to the Redis databases and it will NOT send any commands to the databases. @@ -42,7 +42,7 @@ Copy your service account .json files in the root directory of the project: cp path/to/service_account.json . ``` -Execute +Execute ``` python msstats.py @@ -50,16 +50,16 @@ python msstats.py This generates a file named .xlsx. You need to get that file and send it to Redis. By default, it uses steps of 60 seconds and a period of 7 days (604800 seconds). -You can set different values as follows : +You can set different values as follows: ```` python msstats.py --duration 1800 --step 300 ```` -This can help solving issue like : +This can help solving issue like: ``` -google.api_core.exceptions.ResourceExhausted: 429 Maximum response size of 200000000 bytes reached. +google.api_core.exceptions.ResourceExhausted: 429 Maximum response size of 200000000 bytes reached. Consider querying less data by increasing the step or interval, using more filters and aggregations, or limiting the time duration. ``` @@ -87,8 +87,8 @@ Execute ``` ./batch_run_msstats.sh ``` - -Remove monitoring.viewer role from the service account in all associated Google Cloud projects + +Remove monitoring.viewer role from the service account in all associated Google Cloud projects ``` ./remove_sa_monitoring_viewer.sh @@ -98,6 +98,18 @@ For example, ./remove_sa_monitoring_viewer.sh gmflau-sa@gcp-dev-day-nyc.iam.gserviceaccount.com ``` +## Testing and Code Quality + +Run all tests (unit and integration): +``` +pytest test_msstats.py +``` + +Format code: +``` +black *.py +``` + When finished do not forget to deactivate the virtual environment ``` diff --git a/msstats.py b/msstats.py index b2a6639..0cc62ea 100644 --- a/msstats.py +++ b/msstats.py @@ -33,24 +33,24 @@ def get_all_commands(commands): def processNodeStats(processedMetricPoints): nodeStats = { - 'Throughput (Ops)': 0, - 'GetTypeCmds': 0, - 'SetTypeCmds': 0, - 'OtherTypeCmds': 0, - 'BitmapBasedCmds': 0, - 'ClusterBasedCmds': 0, - 'EvalBasedCmds': 0, - 'GeoSpatialBasedCmds': 0, - 'HashBasedCmds': 0, - 'HyperLogLogBasedCmds': 0, - 'KeyBasedCmds': 0, - 'ListBasedCmds': 0, - 'PubSubBasedCmds': 0, - 'SetBasedCmds': 0, - 'SortedSetBasedCmds': 0, - 'StringBasedCmds': 0, - 'StreamBasedCmds': 0, - 'TransactionBasedCmds': 0 + "Throughput (Ops)": 0, + "GetTypeCmds": 0, + "SetTypeCmds": 0, + "OtherTypeCmds": 0, + "BitmapBasedCmds": 0, + "ClusterBasedCmds": 0, + "EvalBasedCmds": 0, + "GeoSpatialBasedCmds": 0, + "HashBasedCmds": 0, + "HyperLogLogBasedCmds": 0, + "KeyBasedCmds": 0, + "ListBasedCmds": 0, + "PubSubBasedCmds": 0, + "SetBasedCmds": 0, + "SortedSetBasedCmds": 0, + "StringBasedCmds": 0, + "StreamBasedCmds": 0, + "TransactionBasedCmds": 0, } for processedMetricPoint in processedMetricPoints.values(): @@ -64,533 +64,554 @@ def processNodeStats(processedMetricPoints): def processMetricPoint(metricPoint): processedMetricPoint = {} - processedMetricPoint['Throughput (Ops)'] = round(get_all_commands(metricPoint)) + processedMetricPoint["Throughput (Ops)"] = round(get_all_commands(metricPoint)) # Get type commands - processedMetricPoint['GetTypeCmds'] = round(get_command_by_args( - metricPoint, - 'bitcount', - 'bitfield_ro', - 'bitpos', - 'getbit', - 'geodist', - 'geohash', - 'geopos', - 'georadiusbymember_ro', - 'georadius_ro', - 'geosearch', - 'hexists', - 'hget', - 'hgetall', - 'hkeys', - 'hlen', - 'hmget', - 'hrandfield', - 'hscan', - 'hstrlen', - 'hvals', - 'pfcount', - 'dump', - 'exists', - 'expiretime', - 'keys', - 'pexpiretime', - 'pttl', - 'randomkey', - 'scan', - 'sort', - 'sort_ro', - 'touch', - 'ttl', - 'type', - 'lindex', - 'llen', - 'lpos', - 'lrange', - 'scard', - 'sdiff', - 'sinter', - 'sintercard', - 'sismember', - 'smembers', - 'smismember', - 'srandmember', - 'sscan', - 'sunion', - 'zcard', - 'zcount', - 'zdiff', - 'zinter', - 'zintercard', - 'zlexcount', - 'zmscore', - 'zrandmember', - 'zrange', - 'zrangebylex', - 'zrangebyscore', - 'zrank', - 'zrevrange', - 'zrevrank', - 'zscan', - 'zscore', - 'zunion', - 'get', - 'getrange', - 'lcs', - 'mget', - 'strlen', - 'substr', - 'xinfo', - 'xlen', - 'xpending', - 'xrange', - 'xread', - 'xrevrange' - )) + processedMetricPoint["GetTypeCmds"] = round( + get_command_by_args( + metricPoint, + "bitcount", + "bitfield_ro", + "bitpos", + "getbit", + "geodist", + "geohash", + "geopos", + "georadiusbymember_ro", + "georadius_ro", + "geosearch", + "hexists", + "hget", + "hgetall", + "hkeys", + "hlen", + "hmget", + "hrandfield", + "hscan", + "hstrlen", + "hvals", + "pfcount", + "dump", + "exists", + "expiretime", + "keys", + "pexpiretime", + "pttl", + "randomkey", + "scan", + "sort", + "sort_ro", + "touch", + "ttl", + "type", + "lindex", + "llen", + "lpos", + "lrange", + "scard", + "sdiff", + "sinter", + "sintercard", + "sismember", + "smembers", + "smismember", + "srandmember", + "sscan", + "sunion", + "zcard", + "zcount", + "zdiff", + "zinter", + "zintercard", + "zlexcount", + "zmscore", + "zrandmember", + "zrange", + "zrangebylex", + "zrangebyscore", + "zrank", + "zrevrange", + "zrevrank", + "zscan", + "zscore", + "zunion", + "get", + "getrange", + "lcs", + "mget", + "strlen", + "substr", + "xinfo", + "xlen", + "xpending", + "xrange", + "xread", + "xrevrange", + ) + ) # Set type commands - processedMetricPoint['SetTypeCmds'] = round(get_command_by_args( - metricPoint, - 'bitfield', - 'bitop', - 'setbit', - 'geoadd', - 'georadius', - 'georadiusbymember', - 'geosearchstore', - 'hdel', - 'hincrby', - 'hincrbyfloat', - 'hmset', - 'hset', - 'hsetnx', - 'pfadd', - 'pfdebug', - 'pfmerge', - 'copy', - 'del', - 'expire', - 'expireat', - 'migrate', - 'move', - 'persist', - 'pexpire', - 'pexpireat', - 'rename', - 'renamenx', - 'restore', - 'sort', - 'unlink', - 'blmove', - 'blmpop', - 'blpop', - 'brpop', - 'brpoplpush', - 'linsert', - 'lmove', - 'lmpop', - 'lpop', - 'lpush', - 'lpushx', - 'lrem', - 'lset', - 'ltrim', - 'rpop', - 'rpoplpush', - 'rpush', - 'rpushx', - 'sadd', - 'sdiffstore', - 'sinterstore', - 'smove', - 'spop', - 'srem', - 'sunionstore', - 'bzmpop', - 'bzpopmax', - 'bzpopmin', - 'zadd', - 'zdiffstore', - 'zincrby', - 'zinterstore', - 'zmpop', - 'zpopmax', - 'zpopmin', - 'zrangestore', - 'zrem', - 'zremrangebylex', - 'zremrangebyrank', - 'zremrangebyscore', - 'zrevrangebylex', - 'zrevrangebyscore', - 'zunionstore', - 'append', - 'decr', - 'decrby', - 'getdel', - 'getex', - 'getset', - 'incr', - 'incrby', - 'incrbyfloat', - 'mset', - 'msetnx', - 'psetex', - 'set', - 'setex', - 'setnx', - 'setrange', - 'xack', - 'xadd', - 'xautoclaim', - 'xclaim', - 'xdel', - 'xgroup', - 'xreadgroup', - 'xsetid', - 'xtrim' - )) + processedMetricPoint["SetTypeCmds"] = round( + get_command_by_args( + metricPoint, + "bitfield", + "bitop", + "setbit", + "geoadd", + "georadius", + "georadiusbymember", + "geosearchstore", + "hdel", + "hincrby", + "hincrbyfloat", + "hmset", + "hset", + "hsetnx", + "pfadd", + "pfdebug", + "pfmerge", + "copy", + "del", + "expire", + "expireat", + "migrate", + "move", + "persist", + "pexpire", + "pexpireat", + "rename", + "renamenx", + "restore", + "sort", + "unlink", + "blmove", + "blmpop", + "blpop", + "brpop", + "brpoplpush", + "linsert", + "lmove", + "lmpop", + "lpop", + "lpush", + "lpushx", + "lrem", + "lset", + "ltrim", + "rpop", + "rpoplpush", + "rpush", + "rpushx", + "sadd", + "sdiffstore", + "sinterstore", + "smove", + "spop", + "srem", + "sunionstore", + "bzmpop", + "bzpopmax", + "bzpopmin", + "zadd", + "zdiffstore", + "zincrby", + "zinterstore", + "zmpop", + "zpopmax", + "zpopmin", + "zrangestore", + "zrem", + "zremrangebylex", + "zremrangebyrank", + "zremrangebyscore", + "zrevrangebylex", + "zrevrangebyscore", + "zunionstore", + "append", + "decr", + "decrby", + "getdel", + "getex", + "getset", + "incr", + "incrby", + "incrbyfloat", + "mset", + "msetnx", + "psetex", + "set", + "setex", + "setnx", + "setrange", + "xack", + "xadd", + "xautoclaim", + "xclaim", + "xdel", + "xgroup", + "xreadgroup", + "xsetid", + "xtrim", + ) + ) # Other type commands - processedMetricPoint['OtherTypeCmds'] = round(get_command_by_args( - metricPoint, - 'asking', - 'cluster', - 'readonly', - 'readwrite', - 'auth', - 'client', - 'echo', - 'hello', - 'ping', - 'quit', - 'reset', - 'select', - 'eval', - 'evalsha', - 'evalsha_ro', - 'eval_ro', - 'fcall', - 'fcall_ro', - 'function', - 'script', - 'pfselftest', - 'object', - 'wait', - 'psubscribe', - 'publish', - 'pubsub', - 'punsubscribe', - 'spublish', - 'ssubscribe', - 'subscribe', - 'sunsubscribe', - 'unsubscribe', - 'discard', - 'exec', - 'multi', - 'unwatch', - 'watch' - )) + processedMetricPoint["OtherTypeCmds"] = round( + get_command_by_args( + metricPoint, + "asking", + "cluster", + "readonly", + "readwrite", + "auth", + "client", + "echo", + "hello", + "ping", + "quit", + "reset", + "select", + "eval", + "evalsha", + "evalsha_ro", + "eval_ro", + "fcall", + "fcall_ro", + "function", + "script", + "pfselftest", + "object", + "wait", + "psubscribe", + "publish", + "pubsub", + "punsubscribe", + "spublish", + "ssubscribe", + "subscribe", + "sunsubscribe", + "unsubscribe", + "discard", + "exec", + "multi", + "unwatch", + "watch", + ) + ) # Bitmaps based commands - processedMetricPoint['BitmapBasedCmds'] = round(get_command_by_args( - metricPoint, - 'bitcount', - 'bitfield', - 'bitfield_ro', - 'bitop', - 'bitpos', - 'getbit', - 'setbit' - )) + processedMetricPoint["BitmapBasedCmds"] = round( + get_command_by_args( + metricPoint, + "bitcount", + "bitfield", + "bitfield_ro", + "bitop", + "bitpos", + "getbit", + "setbit", + ) + ) # Cluster based commands - processedMetricPoint['ClusterBasedCmds'] = round(get_command_by_args( - metricPoint, - 'asking', - 'cluster', - 'readonly', - 'readwrite' - )) + processedMetricPoint["ClusterBasedCmds"] = round( + get_command_by_args(metricPoint, "asking", "cluster", "readonly", "readwrite") + ) # Eval based commands - processedMetricPoint['EvalBasedCmds'] = round(get_command_by_args( - metricPoint, - 'eval', - 'evalsha', - 'evalsha_ro', - 'eval_ro', - 'fcall', - 'fcall_ro', - 'function', - 'script' - )) + processedMetricPoint["EvalBasedCmds"] = round( + get_command_by_args( + metricPoint, + "eval", + "evalsha", + "evalsha_ro", + "eval_ro", + "fcall", + "fcall_ro", + "function", + "script", + ) + ) # GeoSpatial based commands - processedMetricPoint['GeoSpatialBasedCmds'] = round(get_command_by_args( - metricPoint, - 'geoadd', - 'geodist', - 'geohash', - 'geopos', - 'georadius', - 'georadiusbymember', - 'georadiusbymember_ro', - 'georadius_ro', - 'geosearch', - 'geosearchstore' - )) + processedMetricPoint["GeoSpatialBasedCmds"] = round( + get_command_by_args( + metricPoint, + "geoadd", + "geodist", + "geohash", + "geopos", + "georadius", + "georadiusbymember", + "georadiusbymember_ro", + "georadius_ro", + "geosearch", + "geosearchstore", + ) + ) # Hash based commands - processedMetricPoint['HashBasedCmds'] = round(get_command_by_args( - metricPoint, - 'hdel', - 'hexists', - 'hget', - 'hgetall', - 'hincrby', - 'hincrbyfloat', - 'hkeys', - 'hlen', - 'hmget', - 'hmset', - 'hrandfield', - 'hscan', - 'hset', - 'hsetnx', - 'hstrlen', - 'hvals' - )) + processedMetricPoint["HashBasedCmds"] = round( + get_command_by_args( + metricPoint, + "hdel", + "hexists", + "hget", + "hgetall", + "hincrby", + "hincrbyfloat", + "hkeys", + "hlen", + "hmget", + "hmset", + "hrandfield", + "hscan", + "hset", + "hsetnx", + "hstrlen", + "hvals", + ) + ) # HyperLogLog based commands - processedMetricPoint['HyperLogLogBasedCmds'] = round(get_command_by_args( - metricPoint, - 'pfadd', - 'pfcount', - 'pfdebug', - 'pfmerge', - 'pfselftest' - )) + processedMetricPoint["HyperLogLogBasedCmds"] = round( + get_command_by_args( + metricPoint, "pfadd", "pfcount", "pfdebug", "pfmerge", "pfselftest" + ) + ) # Keys based commands - processedMetricPoint['KeyBasedCmds'] = round(get_command_by_args( - metricPoint, - 'copy', - 'del', - 'dump', - 'exists', - 'expire', - 'expireat', - 'expiretime', - 'keys', - 'migrate', - 'move', - 'object', - 'persist', - 'pexpire', - 'pexpireat', - 'pexpiretime', - 'pttl', - 'randomkey', - 'rename', - 'renamenx', - 'restore', - 'scan', - 'sort', - 'sort_ro', - 'touch', - 'ttl', - 'type', - 'unlink', - 'wait' - )) + processedMetricPoint["KeyBasedCmds"] = round( + get_command_by_args( + metricPoint, + "copy", + "del", + "dump", + "exists", + "expire", + "expireat", + "expiretime", + "keys", + "migrate", + "move", + "object", + "persist", + "pexpire", + "pexpireat", + "pexpiretime", + "pttl", + "randomkey", + "rename", + "renamenx", + "restore", + "scan", + "sort", + "sort_ro", + "touch", + "ttl", + "type", + "unlink", + "wait", + ) + ) # List based commands - processedMetricPoint['ListBasedCmds'] = round(get_command_by_args( - metricPoint, - 'blmove', - 'blmpop', - 'blpop', - 'brpop', - 'brpoplpush', - 'lindex', - 'linsert', - 'llen', - 'lmove', - 'lmpop', - 'lpop', - 'lpos', - 'lpush', - 'lpushx', - 'lrange', - 'lrem', - 'lset', - 'ltrim', - 'rpop', - 'rpoplpush', - 'rpush', - 'rpushx' - )) + processedMetricPoint["ListBasedCmds"] = round( + get_command_by_args( + metricPoint, + "blmove", + "blmpop", + "blpop", + "brpop", + "brpoplpush", + "lindex", + "linsert", + "llen", + "lmove", + "lmpop", + "lpop", + "lpos", + "lpush", + "lpushx", + "lrange", + "lrem", + "lset", + "ltrim", + "rpop", + "rpoplpush", + "rpush", + "rpushx", + ) + ) # PubSub based commands - processedMetricPoint['PubSubBasedCmds'] = round(get_command_by_args( - metricPoint, - 'psubscribe', - 'publish', - 'pubsub', - 'punsubscribe', - 'spublish', - 'ssubscribe', - 'subscribe', - 'sunsubscribe', - 'unsubscribe' - )) + processedMetricPoint["PubSubBasedCmds"] = round( + get_command_by_args( + metricPoint, + "psubscribe", + "publish", + "pubsub", + "punsubscribe", + "spublish", + "ssubscribe", + "subscribe", + "sunsubscribe", + "unsubscribe", + ) + ) # Sets based commands - processedMetricPoint['SetBasedCmds'] = round(get_command_by_args( - metricPoint, - 'sadd', - 'scard', - 'sdiff', - 'sdiffstore', - 'sinter', - 'sintercard', - 'sinterstore', - 'sismember', - 'smembers', - 'smismember', - 'smove', - 'spop', - 'srandmember', - 'srem', - 'sscan', - 'sunion', - 'sunionstore' - )) + processedMetricPoint["SetBasedCmds"] = round( + get_command_by_args( + metricPoint, + "sadd", + "scard", + "sdiff", + "sdiffstore", + "sinter", + "sintercard", + "sinterstore", + "sismember", + "smembers", + "smismember", + "smove", + "spop", + "srandmember", + "srem", + "sscan", + "sunion", + "sunionstore", + ) + ) # SortedSets based commands - processedMetricPoint['SortedSetBasedCmds'] = round(get_command_by_args( - metricPoint, - 'bzmpop', - 'bzpopmax', - 'bzpopmin', - 'zadd', - 'zcard', - 'zcount', - 'zdiff', - 'zdiffstore', - 'zincrby', - 'zinter', - 'zintercard', - 'zinterstore', - 'zlexcount', - 'zmpop', - 'zmscore', - 'zpopmax', - 'zpopmin', - 'zrandmember', - 'zrange', - 'zrangebylex', - 'zrangebyscore', - 'zrangestore', - 'zrank', - 'zrem', - 'zremrangebylex', - 'zremrangebyrank', - 'zremrangebyscore', - 'zrevrange', - 'zrevrangebylex', - 'zrevrangebyscore', - 'zrevrank', - 'zscan', - 'zscore', - 'zunion', - 'zunionstore' - )) + processedMetricPoint["SortedSetBasedCmds"] = round( + get_command_by_args( + metricPoint, + "bzmpop", + "bzpopmax", + "bzpopmin", + "zadd", + "zcard", + "zcount", + "zdiff", + "zdiffstore", + "zincrby", + "zinter", + "zintercard", + "zinterstore", + "zlexcount", + "zmpop", + "zmscore", + "zpopmax", + "zpopmin", + "zrandmember", + "zrange", + "zrangebylex", + "zrangebyscore", + "zrangestore", + "zrank", + "zrem", + "zremrangebylex", + "zremrangebyrank", + "zremrangebyscore", + "zrevrange", + "zrevrangebylex", + "zrevrangebyscore", + "zrevrank", + "zscan", + "zscore", + "zunion", + "zunionstore", + ) + ) # String based commands - processedMetricPoint['StringBasedCmds'] = round(get_command_by_args( - metricPoint, - 'append', - 'decr', - 'decrby', - 'get', - 'getdel', - 'getex', - 'getrange', - 'getset', - 'incr', - 'incrby', - 'incrbyfloat', - 'lcs', - 'mget', - 'mset', - 'msetnx', - 'psetex', - 'set', - 'setex', - 'setnx', - 'setrange', - 'strlen', - 'substr' - )) + processedMetricPoint["StringBasedCmds"] = round( + get_command_by_args( + metricPoint, + "append", + "decr", + "decrby", + "get", + "getdel", + "getex", + "getrange", + "getset", + "incr", + "incrby", + "incrbyfloat", + "lcs", + "mget", + "mset", + "msetnx", + "psetex", + "set", + "setex", + "setnx", + "setrange", + "strlen", + "substr", + ) + ) # Stream based commands - processedMetricPoint['StreamBasedCmds'] = round(get_command_by_args( - metricPoint, - 'xack', - 'xadd', - 'xautoclaim', - 'xclaim', - 'xdel', - 'xgroup', - 'xinfo', - 'xlen', - 'xpending', - 'xrange', - 'xread', - 'xreadgroup', - 'xrevrange', - 'xsetid', - 'xtrim' - )) + processedMetricPoint["StreamBasedCmds"] = round( + get_command_by_args( + metricPoint, + "xack", + "xadd", + "xautoclaim", + "xclaim", + "xdel", + "xgroup", + "xinfo", + "xlen", + "xpending", + "xrange", + "xread", + "xreadgroup", + "xrevrange", + "xsetid", + "xtrim", + ) + ) # Transaction based commands - processedMetricPoint['TransactionBasedCmds'] = round(get_command_by_args( - metricPoint, - 'discard', - 'exec', - 'multi', - 'unwatch', - 'watch' - )) + processedMetricPoint["TransactionBasedCmds"] = round( + get_command_by_args(metricPoint, "discard", "exec", "multi", "unwatch", "watch") + ) return processedMetricPoint -def process_google_service_account(service_account, project_id, duration=604800, step=60): +def process_google_service_account( + service_account, project_id, duration=604800, step=60 +): if not project_id: try: f = open(service_account, "r") data = json.loads(f.read()) f.close() - project_id = data['project_id'] + project_id = data["project_id"] if not project_id: raise Exception("Invalid json file") except: return # Set the value GOOGLE_APPLICATION_CREDENTIALS variable - os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = service_account - print("Processing Google Account with credentials found in: ", os.environ['GOOGLE_APPLICATION_CREDENTIALS']) + os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = service_account + print( + "Processing Google Account with credentials found in: ", + os.environ["GOOGLE_APPLICATION_CREDENTIALS"], + ) client = monitoring_v3.MetricServiceClient() project_name = f"projects/{project_id}" now = time.time() seconds = int(now) - nanos = int((now - seconds) * 10 ** 9) + nanos = int((now - seconds) * 10**9) interval = monitoring_v3.TimeInterval( { @@ -602,7 +623,7 @@ def process_google_service_account(service_account, project_id, duration=604800, { "alignment_period": {"seconds": step}, "per_series_aligner": monitoring_v3.Aggregation.Aligner.ALIGN_RATE, - "cross_series_reducer": monitoring_v3.Aggregation.Reducer.REDUCE_NONE + "cross_series_reducer": monitoring_v3.Aggregation.Reducer.REDUCE_NONE, } ) @@ -625,7 +646,7 @@ def process_google_service_account(service_account, project_id, duration=604800, for result in results: database = extractDatabaseName(result.resource.labels["instance_id"]) node_id = result.resource.labels["node_id"] - cmd = result.metric.labels['cmd'] + cmd = result.metric.labels["cmd"] if not database in metric_points: metric_points[database] = {} @@ -634,12 +655,14 @@ def process_google_service_account(service_account, project_id, duration=604800, "Source": "MS", "ClusterId": database, "NodeId": node_id, - "NodeRole": "Master" if result.metric.labels['role'] == 'primary' else "Replica", + "NodeRole": ( + "Master" if result.metric.labels["role"] == "primary" else "Replica" + ), "NodeType": "", "Region": result.resource.labels["region"], "Project ID": project_id, "InstanceId": result.resource.labels["instance_id"], - "points": {} + "points": {}, } for point in result.points: @@ -648,9 +671,9 @@ def process_google_service_account(service_account, project_id, duration=604800, metric_points[database][node_id]["points"][interval] = {} point_value = 0 - if (result.value_type == 2): + if result.value_type == 2: point_value = point.value.int64_value - elif (result.value_type == 3): + elif result.value_type == 3: point_value = point.value.double_value metric_points[database][node_id]["points"][interval][cmd] = point_value @@ -660,13 +683,15 @@ def process_google_service_account(service_account, project_id, duration=604800, if not "processed_points" in metric_points[database][node]: metric_points[database][node]["processed_points"] = {} for point in metric_points[database][node]["points"]: - metric_points[database][node]["processed_points"][point] = processMetricPoint( - metric_points[database][node]["points"][point]) + metric_points[database][node]["processed_points"][point] = ( + processMetricPoint(metric_points[database][node]["points"][point]) + ) for database in metric_points: for node in metric_points[database]: - metric_points[database][node]['commandstats'] = processNodeStats( - metric_points[database][node]["processed_points"]) + metric_points[database][node]["commandstats"] = processNodeStats( + metric_points[database][node]["processed_points"] + ) del metric_points[database][node]["points"] del metric_points[database][node]["processed_points"] @@ -675,7 +700,7 @@ def process_google_service_account(service_account, project_id, duration=604800, interval = monitoring_v3.TimeInterval() now = time.time() seconds = int(now) - nanos = int((now - seconds) * 10 ** 9) + nanos = int((now - seconds) * 10**9) interval = monitoring_v3.TimeInterval( { "end_time": {"seconds": seconds, "nanos": nanos}, @@ -699,7 +724,7 @@ def process_google_service_account(service_account, project_id, duration=604800, if point.value.int64_value > BytesUsedForCache: BytesUsedForCache = point.value.int64_value if database in metric_points: - metric_points[database][node_id]['BytesUsedForCache'] = BytesUsedForCache + metric_points[database][node_id]["BytesUsedForCache"] = BytesUsedForCache # Retrieve MaxMemory (a.k.a. Capacity) results = client.list_time_series( @@ -718,7 +743,7 @@ def process_google_service_account(service_account, project_id, duration=604800, if point.value.int64_value > MaxMemory: MaxMemory = point.value.int64_value if database in metric_points: - metric_points[database][node_id]['MaxMemory'] = MaxMemory + metric_points[database][node_id]["MaxMemory"] = MaxMemory # CacheHits # CacheMisses @@ -740,14 +765,14 @@ def create_workbooks(outDir, projects): for project in projects: wb = openpyxl.Workbook() ws = wb.active - ws.title = 'ClusterData' + ws.title = "ClusterData" for cluster in projects[project]: for node in projects[project][cluster]: - node_commandstats = projects[project][cluster][node]['commandstats'] + node_commandstats = projects[project][cluster][node]["commandstats"] node_info = projects[project][cluster][node] - del node_info['commandstats'] + del node_info["commandstats"] node_stats = {**node_info, **node_commandstats} if node_stats is not None: @@ -772,7 +797,7 @@ def main(): dest="outDir", default=".", help="The directory to output the results. If the directory does not exist the script will try to create it.", - metavar="PATH" + metavar="PATH", ) parser.add_option( @@ -781,7 +806,7 @@ def main(): dest="project_id", default="", help="The Google Cloud Project ID containing MemoryStore instances.", - metavar="PROJECT_ID" + metavar="PROJECT_ID", ) parser.add_option( @@ -789,14 +814,14 @@ def main(): dest="step", type="int", default=60, - help="Step (alignment_period) in seconds between data points. Default is 60." + help="Step (alignment_period) in seconds between data points. Default is 60.", ) parser.add_option( "--duration", dest="duration", type="int", default=604800, - help="Duration of the metric window in seconds. Default is 604800 (7 days)." + help="Duration of the metric window in seconds. Default is 604800 (7 days).", ) (options, _) = parser.parse_args() @@ -805,18 +830,20 @@ def main(): os.makedirs(options.outDir) # Scan for .json files in order to find the service account files - path_to_json = '.' - service_accounts = [os.path.abspath(os.path.join(path_to_json, pos_json)) for pos_json in os.listdir(path_to_json) - if pos_json.endswith('.json')] + path_to_json = "." + service_accounts = [ + os.path.abspath(os.path.join(path_to_json, pos_json)) + for pos_json in os.listdir(path_to_json) + if pos_json.endswith(".json") + ] projects = {} # For each service account found try to fetch the clusters metrics using the # google cloud monitoring api metrics for service_account in service_accounts: - project_id, stats = project_id, stats = process_google_service_account(service_account, - options.project_id, - options.duration, - options.step) + project_id, stats = project_id, stats = process_google_service_account( + service_account, options.project_id, options.duration, options.step + ) projects[project_id] = stats create_workbooks(options.outDir, projects) diff --git a/requirements.txt b/requirements.txt index efe9f02..cbd6e53 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ openpyxl>=3.0.4 google-cloud-monitoring==2.18.0 +black>=25.1.0 +pytest>=8.4.0 +pytest-mock>=3.14.0 diff --git a/test_msstats.py b/test_msstats.py new file mode 100644 index 0000000..b282e9e --- /dev/null +++ b/test_msstats.py @@ -0,0 +1,480 @@ +import unittest +import os +import tempfile +import json +from unittest.mock import patch, MagicMock +import openpyxl + +from msstats import ( + extractDatabaseName, + get_command_by_args, + get_all_commands, + processNodeStats, + processMetricPoint, + process_google_service_account, + create_workbooks, +) + + +class TestMSSStats(unittest.TestCase): + """Test suite for msstats utility functions""" + + def test_extractDatabaseName(self): + """Test extracting database name from instance ID""" + # Test typical instance ID format + instance_id = ( + "projects/test-project/locations/us-central1/instances/redis-instance" + ) + expected = "redis-instance" + result = extractDatabaseName(instance_id) + self.assertEqual(result, expected) + + # Test with different format + instance_id = "projects/my-project/locations/europe-west1/instances/my-redis-db" + expected = "my-redis-db" + result = extractDatabaseName(instance_id) + self.assertEqual(result, expected) + + # Test edge case with single part + instance_id = "redis-simple" + expected = "redis-simple" + result = extractDatabaseName(instance_id) + self.assertEqual(result, expected) + + def test_get_command_by_args(self): + """Test getting command counts by specific arguments""" + commands = {"get": 100, "set": 50, "del": 25, "incr": 10, "exists": 5} + + # Test single command + result = get_command_by_args(commands, "get") + self.assertEqual(result, 100) + + # Test multiple commands + result = get_command_by_args(commands, "get", "set") + self.assertEqual(result, 150) + + # Test with non-existent command + result = get_command_by_args(commands, "nonexistent") + self.assertEqual(result, 0) + + # Test mix of existing and non-existent commands + result = get_command_by_args(commands, "get", "nonexistent", "set") + self.assertEqual(result, 150) + + # Test empty args + result = get_command_by_args(commands) + self.assertEqual(result, 0) + + def test_get_all_commands(self): + """Test getting total count of all commands""" + commands = {"get": 100, "set": 50, "del": 25, "incr": 10} + + result = get_all_commands(commands) + self.assertEqual(result, 185) + + # Test empty commands dict + result = get_all_commands({}) + self.assertEqual(result, 0) + + # Test single command + result = get_all_commands({"test": 42}) + self.assertEqual(result, 42) + + def test_processNodeStats(self): + """Test processing node statistics to get maximum values""" + # Create sample processed metric points from multiple nodes + processed_metric_points = { + "node1": { + "Throughput (Ops)": 100, + "GetTypeCmds": 50, + "SetTypeCmds": 30, + "OtherTypeCmds": 20, + "BitmapBasedCmds": 0, + "ClusterBasedCmds": 0, + "EvalBasedCmds": 0, + "GeoSpatialBasedCmds": 0, + "HashBasedCmds": 10, + "HyperLogLogBasedCmds": 0, + "KeyBasedCmds": 5, + "ListBasedCmds": 0, + "PubSubBasedCmds": 0, + "SetBasedCmds": 0, + "SortedSetBasedCmds": 0, + "StringBasedCmds": 15, + "StreamBasedCmds": 0, + "TransactionBasedCmds": 0, + }, + "node2": { + "Throughput (Ops)": 150, # Higher than node1 + "GetTypeCmds": 40, + "SetTypeCmds": 60, # Higher than node1 + "OtherTypeCmds": 10, + "BitmapBasedCmds": 5, # Higher than node1 + "ClusterBasedCmds": 0, + "EvalBasedCmds": 0, + "GeoSpatialBasedCmds": 0, + "HashBasedCmds": 8, + "HyperLogLogBasedCmds": 0, + "KeyBasedCmds": 12, # Higher than node1 + "ListBasedCmds": 0, + "PubSubBasedCmds": 0, + "SetBasedCmds": 0, + "SortedSetBasedCmds": 0, + "StringBasedCmds": 10, + "StreamBasedCmds": 0, + "TransactionBasedCmds": 0, + }, + } + + result = processNodeStats(processed_metric_points) + + # Should take maximum values across all nodes + self.assertEqual(result["Throughput (Ops)"], 150) # max from node2 + self.assertEqual(result["GetTypeCmds"], 50) # max from node1 + self.assertEqual(result["SetTypeCmds"], 60) # max from node2 + self.assertEqual(result["BitmapBasedCmds"], 5) # max from node2 + self.assertEqual(result["KeyBasedCmds"], 12) # max from node2 + + def test_processMetricPoint(self): + """Test processing a single metric point""" + # Create sample metric point data + metric_point = { + "get": 50, + "set": 30, + "hget": 20, # Hash command + "lpush": 10, # List command + "sadd": 5, # Set command + "unknown_cmd": 15, # Should be counted in total but not categorized + } + + result = processMetricPoint(metric_point) + + # Check that all commands are counted in throughput + expected_throughput = sum(metric_point.values()) + self.assertEqual(result["Throughput (Ops)"], expected_throughput) + + # Check that hash commands are properly categorized + self.assertEqual(result["HashBasedCmds"], 20) # hget + + # Check that list commands are properly categorized + self.assertEqual(result["ListBasedCmds"], 10) # lpush + + # Check that set commands are properly categorized + self.assertEqual(result["SetBasedCmds"], 5) # sadd + + # Verify structure - all expected keys should be present + expected_keys = { + "Throughput (Ops)", + "GetTypeCmds", + "SetTypeCmds", + "OtherTypeCmds", + "BitmapBasedCmds", + "ClusterBasedCmds", + "EvalBasedCmds", + "GeoSpatialBasedCmds", + "HashBasedCmds", + "HyperLogLogBasedCmds", + "KeyBasedCmds", + "ListBasedCmds", + "PubSubBasedCmds", + "SetBasedCmds", + "SortedSetBasedCmds", + "StringBasedCmds", + "StreamBasedCmds", + "TransactionBasedCmds", + } + self.assertEqual(set(result.keys()), expected_keys) + + def test_processMetricPoint_empty(self): + """Test processing empty metric point""" + result = processMetricPoint({}) + + # All values should be 0 + for value in result.values(): + self.assertEqual(value, 0) + + def test_processNodeStats_single_node(self): + """Test processing node stats with single node""" + processed_metric_points = { + "node1": { + "Throughput (Ops)": 100, + "GetTypeCmds": 50, + "SetTypeCmds": 30, + "OtherTypeCmds": 20, + "BitmapBasedCmds": 0, + "ClusterBasedCmds": 0, + "EvalBasedCmds": 0, + "GeoSpatialBasedCmds": 0, + "HashBasedCmds": 10, + "HyperLogLogBasedCmds": 0, + "KeyBasedCmds": 5, + "ListBasedCmds": 0, + "PubSubBasedCmds": 0, + "SetBasedCmds": 0, + "SortedSetBasedCmds": 0, + "StringBasedCmds": 15, + "StreamBasedCmds": 0, + "TransactionBasedCmds": 0, + } + } + + result = processNodeStats(processed_metric_points) + + # Should return the same values as the single node + self.assertEqual(result["Throughput (Ops)"], 100) + self.assertEqual(result["GetTypeCmds"], 50) + self.assertEqual(result["HashBasedCmds"], 10) + + def test_processNodeStats_empty(self): + """Test processing empty node stats""" + result = processNodeStats({}) + + # Should return default structure with all zeros + expected_keys = { + "Throughput (Ops)", + "GetTypeCmds", + "SetTypeCmds", + "OtherTypeCmds", + "BitmapBasedCmds", + "ClusterBasedCmds", + "EvalBasedCmds", + "GeoSpatialBasedCmds", + "HashBasedCmds", + "HyperLogLogBasedCmds", + "KeyBasedCmds", + "ListBasedCmds", + "PubSubBasedCmds", + "SetBasedCmds", + "SortedSetBasedCmds", + "StringBasedCmds", + "StreamBasedCmds", + "TransactionBasedCmds", + } + + self.assertEqual(set(result.keys()), expected_keys) + for value in result.values(): + self.assertEqual(value, 0) + + +class TestMSStatsIntegration(unittest.TestCase): + """Integration tests for msstats functionality""" + + def setUp(self): + """Set up test fixtures""" + self.temp_dir = tempfile.mkdtemp() + self.test_project_id = "test-project-123" + + # Create a mock service account file with required fields + self.service_account_data = { + "project_id": self.test_project_id, + "type": "service_account", + "private_key_id": "test-key-id", + "private_key": "Some Mock Private Key\n", + "client_email": "test@test-project.iam.gserviceaccount.com", + "client_id": "123456789", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + } + + self.service_account_file = os.path.join( + self.temp_dir, "test-service-account.json" + ) + with open(self.service_account_file, "w") as f: + json.dump(self.service_account_data, f) + + def tearDown(self): + """Clean up test fixtures""" + if os.path.exists(self.service_account_file): + os.remove(self.service_account_file) + os.rmdir(self.temp_dir) + + @patch("msstats.monitoring_v3.MetricServiceClient") + def test_process_google_service_account_with_mock_data(self, mock_client_class): + """Test processing service account with mocked Google Cloud responses""" + # Mock the monitoring client + mock_client = MagicMock() + mock_client_class.return_value = mock_client + + # Mock time series response for commands/calls + mock_result = MagicMock() + mock_result.resource.labels = MagicMock() + mock_result.resource.labels.__getitem__ = MagicMock( + side_effect=lambda key: { + "instance_id": "projects/test-project/locations/us-central1/instances/test-redis", + "region": "us-central1", + "node_id": "node-0", + }[key] + ) + + mock_result.metric.labels = MagicMock() + mock_result.metric.labels.__getitem__ = MagicMock( + side_effect=lambda key: {"cmd": "get", "role": "primary"}[key] + ) + + # Mock data points + mock_point = MagicMock() + mock_point.interval.start_time.timestamp.return_value = ( + 1640995200.0 # 2022-01-01 + ) + mock_point.value.int64_value = 100 + mock_result.points = [mock_point] + mock_result.value_type = 2 # INT64 + + # Mock memory usage response + mock_memory_result = MagicMock() + mock_memory_result.resource.labels = MagicMock() + mock_memory_result.resource.labels.__getitem__ = MagicMock( + side_effect=lambda key: { + "instance_id": "projects/test-project/locations/us-central1/instances/test-redis", + "node_id": "node-0", + }[key] + ) + mock_memory_point = MagicMock() + mock_memory_point.value.int64_value = 1000000 # 1MB + mock_memory_result.points = [mock_memory_point] + + # Set up client responses + mock_client.list_time_series.side_effect = [ + [mock_result], # commands/calls response + [mock_memory_result], # memory/usage response + [mock_memory_result], # memory/maxmemory response + ] + + # Test the function + project_id, stats = process_google_service_account( + self.service_account_file, + self.test_project_id, + duration=3600, # 1 hour + step=60, + ) + + # Assertions + self.assertEqual(project_id, self.test_project_id) + self.assertIsInstance(stats, dict) + self.assertIn("test-redis", stats) + + # Check that the database stats contain expected structure + database_stats = stats["test-redis"] + self.assertIn("node-0", database_stats) + + node_stats = database_stats["node-0"] + self.assertIn("commandstats", node_stats) + self.assertIn("Source", node_stats) + self.assertIn("ClusterId", node_stats) + self.assertIn("NodeRole", node_stats) + self.assertEqual(node_stats["NodeRole"], "Master") + + def test_create_workbooks_integration(self): + """Test creating Excel workbooks from processed data""" + # Create sample processed data + projects_data = { + "test-project": { + "redis-instance-1": { + "node-0": { + "Source": "MS", + "ClusterId": "redis-instance-1", + "NodeId": "node-0", + "NodeRole": "Master", + "NodeType": "", + "Region": "us-central1", + "Project ID": "test-project", + "InstanceId": "projects/test-project/locations/us-central1/instances/redis-instance-1", + "BytesUsedForCache": 1000000, + "MaxMemory": 2000000, + "commandstats": { + "Throughput (Ops)": 500, + "GetTypeCmds": 300, + "SetTypeCmds": 200, + "OtherTypeCmds": 0, + "BitmapBasedCmds": 0, + "ClusterBasedCmds": 0, + "EvalBasedCmds": 0, + "GeoSpatialBasedCmds": 0, + "HashBasedCmds": 0, + "HyperLogLogBasedCmds": 0, + "KeyBasedCmds": 0, + "ListBasedCmds": 0, + "PubSubBasedCmds": 0, + "SetBasedCmds": 0, + "SortedSetBasedCmds": 0, + "StringBasedCmds": 0, + "StreamBasedCmds": 0, + "TransactionBasedCmds": 0, + }, + } + } + } + } + + # Create workbooks + create_workbooks(self.temp_dir, projects_data) + + # Verify Excel file was created + excel_file = os.path.join(self.temp_dir, "test-project.xlsx") + self.assertTrue(os.path.exists(excel_file)) + + # Verify Excel file contents + wb = openpyxl.load_workbook(excel_file) + ws = wb.active + self.assertEqual(ws.title, "ClusterData") + + # Check that we have headers and data + self.assertGreater(ws.max_row, 1) # Should have header + at least 1 data row + self.assertGreater(ws.max_column, 10) # Should have many columns + + # Check some expected headers are present + headers = [cell.value for cell in ws[1]] + self.assertIn("Source", headers) + self.assertIn("ClusterId", headers) + self.assertIn("NodeRole", headers) + self.assertIn("Throughput (Ops)", headers) + + # Clean up + wb.close() + os.remove(excel_file) + + @patch("msstats.monitoring_v3.MetricServiceClient") + def test_service_account_file_parsing(self, mock_client_class): + """Test that service account files are parsed correctly""" + # Mock the monitoring client + mock_client = MagicMock() + mock_client_class.return_value = mock_client + mock_client.list_time_series.return_value = [] + + # Test with explicit project_id + project_id, _ = process_google_service_account( + self.service_account_file, "explicit-project-id" + ) + self.assertEqual(project_id, "explicit-project-id") + + # Test without explicit project_id (should read from file) + project_id, _ = process_google_service_account( + self.service_account_file, "" # Empty project_id should read from file + ) + self.assertEqual(project_id, self.test_project_id) + + def test_invalid_service_account_file(self): + """Test handling of invalid service account files""" + # Create invalid JSON file + invalid_file = os.path.join(self.temp_dir, "invalid.json") + with open(invalid_file, "w") as f: + f.write("invalid json content") + + # Should return None for invalid files + result = process_google_service_account(invalid_file, "") + self.assertIsNone(result) + + # Clean up + os.remove(invalid_file) + + def test_missing_service_account_file(self): + """Test handling of missing service account files""" + nonexistent_file = os.path.join(self.temp_dir, "nonexistent.json") + + # Should return None for missing files + result = process_google_service_account(nonexistent_file, "") + self.assertIsNone(result) + + +if __name__ == "__main__": + unittest.main()