From 26f3e39449fa8e18499b929e2fee2855cd29397a Mon Sep 17 00:00:00 2001 From: himanshiparnami Date: Mon, 31 Oct 2022 23:10:58 +0530 Subject: [PATCH 1/2] Adding tag validator utility --- Dockerfile | 13 + Makefile | 8 + README.md | 87 ++++- config/tagvalidator.yml | 37 +++ requirements.txt | 8 + .../__pycache__/generate_html.cpython-38.pyc | Bin 0 -> 3643 bytes scripts/generate_html.py | 163 +++++++++ scripts/tagvalidator.py | 310 ++++++++++++++++++ 8 files changed, 625 insertions(+), 1 deletion(-) create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 config/tagvalidator.yml create mode 100644 requirements.txt create mode 100644 scripts/__pycache__/generate_html.cpython-38.pyc create mode 100755 scripts/generate_html.py create mode 100755 scripts/tagvalidator.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6cdd2c4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM python:slim-buster AS builder + +WORKDIR /opt/ +COPY . . +RUN pip3 install --no-cache --upgrade -r requirements.txt +RUN apt-get update +RUN apt-get install -y binutils libc-bin +RUN pyinstaller scripts/tagvalidator.py --onefile + +FROM python:slim-buster AS deployer +WORKDIR /opt +COPY --from=builder /opt/dist/tagvalidator /usr/local/bin/tagvalidator +ENTRYPOINT ["/usr/local/bin/tagvalidator","-p" , "tagvalidator.yml", "-r", "."] \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..39840c7 --- /dev/null +++ b/Makefile @@ -0,0 +1,8 @@ +build: + docker build -t opstree/tagvalidator:0.1 . + +run: + docker run -it --rm --name tagvalidator -v ~/.aws:/root/.aws -v ${data_dir}:/opt/ opstree/tagvalidator:0.1 + +run-debug: + docker run -it --rm --name tagvalidator -v ~/.aws:/root/.aws/ -v ${data_dir}:/opt/ --entrypoint sh opstree/tagvalidator:0.1 \ No newline at end of file diff --git a/README.md b/README.md index e445c85..143ea15 100644 --- a/README.md +++ b/README.md @@ -1 +1,86 @@ -# BP-VALIDATOR \ No newline at end of file +# BP-VALIDATOR + +As **VALIDATOR** has capability to scan AWS specific resource's tags as per user-defined property file and generates a CSV and HTML report. + +## SERVICES SUPPORTED +- EC2 +- Route53 +- S3 +- RDS +- DynamoDB + +## CONFIGURATIONS +Configuration for this utility will be managed in YAML format. Below are the configurations details : + +- ***aws_profile :*** It is a aws profile that you can use to a perform utility in AWS. + +- ***region :*** The AWS Region where this utility is executed. + +- ***services:ec2:key:value (Optional) :*** Tags given to ec2 instances.It will validate all the ec2 matches to this given tags. + +- ***services:s3:key:value (Optional) :*** Tags given to s3.It will validate all the s3 matches to this given tags. + +- ***services:route53:key:value (Optional) :*** Tags given to route53.It will validate all the route53 matches to this given tags. + +- ***services:dynamodb:key:value (Optional) :*** Tags given to dynamodb.It will validate all the dynamodb matches to this given tags. + +- ***services:rds:key:value (Optional) :*** Tags given to rds .It will validate all the rds matches to this given tags. + +## SAMPLE CONF FILE +``` +case_insensitive: true + +aws_profile: default + +region: + - us-east-1 + +services: + ec2: + env: + - any + learner: + - any + + s3: + learner: + - any + env: + - any + + route53: + learner: + - any + env: + - any + + dynamodb: + env: + - any + learner: + - any + + rds: + learner: + - any + env: + - any +``` + +## USAGE + +=============================================================== + +Three things are needed to use this : +- AWS resources access (either via AWS Access and Secret keys or IAM ROLE). +- YAML Property file +- Reports Path where you want to store HTML ans CSV reports + +### LOCALLY +To run this utility locally from your system. + +- Run the python script. + + ``` + python3 ./scripts/tagvalidator.py -p -r + ``` \ No newline at end of file diff --git a/config/tagvalidator.yml b/config/tagvalidator.yml new file mode 100644 index 0000000..1780f77 --- /dev/null +++ b/config/tagvalidator.yml @@ -0,0 +1,37 @@ +case_insensitive: true + +aws_profile: default + +region: + - us-east-1 + +services: + ec2: + env: + - any + learner: + - any + + s3: + learner: + - any + env: + - any + + route53: + learner: + - any + env: + - any + + dynamodb: + env: + - any + learner: + - any + + rds: + learner: + - any + env: + - any \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..61757dc --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +setuptools +pyyaml +wheel +pandas +pyinstaller +boto3 +git+https://github.com/OT-PYTHON-UTILS/ot-filesystem-libs.git +git+https://github.com/OT-PYTHON-UTILS/ot-aws-libs.git@develop \ No newline at end of file diff --git a/scripts/__pycache__/generate_html.cpython-38.pyc b/scripts/__pycache__/generate_html.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..84073a819aba983268a1bff8bede324bd4bd3801 GIT binary patch literal 3643 zcmb_f-ESMk7GFC~6D(2?ssvAH?iLgq)oUkl(&E^@H6az1(v;he3(D11G~O9|r`g@v zJ2Q@()Yg4~cX;3d1W4^8&-@Mi6MjL0|AHsZ%L^PBTE=Nvy=TPs>H z$WO=ak876otNzH2Jbc`QOTG>_VjWo|NAjOrM>(=|-zpadKLdNQDb<-z+r$*%UO`g-Jyps|xm6KWreEhKh>a;<$(n10N`#g+LzikWPsSb-BQbao|W7 z2$PYI&l-q@nxoS7xqG5HhKwk$fi`RJ>aPHcJ_{>?^*z-`O`^wW1Fo9Zpokm5e`54-xz9xdUESQ--l}aRtEWsdMadjNed~H> zV<#c#ad#lia+eE2MWxG?;sF>B_K@6XM2pnL?nz(pK}cYO4?6qx_irTp9zUhxukr7F zcx%7YN%*m=*eQ(xvYVt@Ae8DgV{29uk2j?&Sfr3Ti)hQ%_N_j`r#POqkrZyr_LPbm z)vC)0bsqh1K*gEk@}L^ORyLgL&W01P(0L^7HUJyIHpEo&0`3@uAR`Y^kNFTd9if=( zQ&~ErSllvG)tGss*u=;awAZq) z+K7j)&)k!iJ#%$q!8Pk(xb^a`-G(Q+w}1cc!+U%C_nTGRMkDl>!G8JfYXf$8_pq{e z_s+)$w>yUq{tno$4D1Uai_^SzlLZkM3Yk!r5X(A2NT<(v-={cY(gd_Cw2wb?-n!RsbeEfRB+U3pwkIVK-x-j88AkVwI4D(YVM#b9%Zrw zx=Ki#DqfIA5`uX^&qU8Ouf~aXo+Qp)Q<5a-F3>;#2&pQr$NZQ+NtYEPq_eH7z#tduAbtZ+qnWn?!HSUSn#!z8bFxo?b<7CGOLbEvRl-?Q6(>&o-6Z=n!x27NGS_#|pf>0@of~@=LglmY!N8>sfAG z7!{u8M!9FXoTc)k+-JEjE{~T-%TJe{S>xqTtZnOG);;S}3uX$shklkfs|B(|3e)#x za%uXWCo7};w-#Cbeo4=1O!CTS`7f@FFO4oeT~?Py%WuNIIctVZEa=ILhvzHJxC1(W zD|t!UplNTkK6n*GZz|G;(%P)Tb8s`BK&G*4t;Y_%dh1pV#(KL6%@lgRnTpwA4H`EM zGDH39WvI2ip~}cp{ov!uzdHxFKkPiXdoMW|vf8I1g+7BCJ)G!lvNkvK z4eTVZz|G1P&F_uetLAqF?&39=KP>0PDm=}Hkt}n&3=5OhX28h+{@^Xam(-`B@PF&g F-vM~7Q#1eo literal 0 HcmV?d00001 diff --git a/scripts/generate_html.py b/scripts/generate_html.py new file mode 100755 index 0000000..2b3691e --- /dev/null +++ b/scripts/generate_html.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 + +HTML = """ + + + + +ALL RESOURCES | +NON-COMPLIANT RESOURCES | +COMPLIANT RESOURCES + + + +""" + +def generate_html(csv_name,html_file_name, case_insensitive, pd): + df = pd.read_csv(csv_name) + df.to_html(html_file_name) + with open(html_file_name) as file: + file = file.read() + file = file.replace("✘","") + file = file.replace("","") + file = file.replace("
✘
✔
✔
Case Insensitive : {case_insensitive}

Date: Mon, 31 Oct 2022 23:14:59 +0530 Subject: [PATCH 2/2] Update README.md --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 143ea15..4b40ac5 100644 --- a/README.md +++ b/README.md @@ -16,15 +16,15 @@ Configuration for this utility will be managed in YAML format. Below are the con - ***region :*** The AWS Region where this utility is executed. -- ***services:ec2:key:value (Optional) :*** Tags given to ec2 instances.It will validate all the ec2 matches to this given tags. +- ***services:ec2 (Optional) :*** Tags given to ec2 instances.It will validate all the ec2 matches to this given tags. -- ***services:s3:key:value (Optional) :*** Tags given to s3.It will validate all the s3 matches to this given tags. +- ***services:s3 (Optional) :*** Tags given to s3.It will validate all the s3 matches to this given tags. -- ***services:route53:key:value (Optional) :*** Tags given to route53.It will validate all the route53 matches to this given tags. +- ***services:route53 (Optional) :*** Tags given to route53.It will validate all the route53 matches to this given tags. -- ***services:dynamodb:key:value (Optional) :*** Tags given to dynamodb.It will validate all the dynamodb matches to this given tags. +- ***services:dynamodb (Optional) :*** Tags given to dynamodb.It will validate all the dynamodb matches to this given tags. -- ***services:rds:key:value (Optional) :*** Tags given to rds .It will validate all the rds matches to this given tags. +- ***services:rds (Optional) :*** Tags given to rds .It will validate all the rds matches to this given tags. ## SAMPLE CONF FILE ``` @@ -83,4 +83,4 @@ To run this utility locally from your system. ``` python3 ./scripts/tagvalidator.py -p -r - ``` \ No newline at end of file + ```