From df62a8d9a170ace19726eba4b398742035af72be Mon Sep 17 00:00:00 2001 From: ege Date: Sun, 7 Mar 2021 20:58:09 +0300 Subject: [PATCH 1/2] added parameter; encoding error response --- core/mirror.py | 4 ++-- core/utils.py | 10 +++++----- photon.py | 11 ++++++----- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/core/mirror.py b/core/mirror.py index 8dfebe0..cafea3a 100644 --- a/core/mirror.py +++ b/core/mirror.py @@ -1,7 +1,7 @@ import os -def mirror(url, response): +def mirror(url, response,encoding_error_response): if response != 'dummy': clean_url = url.replace('http://', '').replace('https://', '').rstrip('/') parts = clean_url.split('?')[0].split('/') @@ -36,4 +36,4 @@ def mirror(url, response): if len(url.split('?')) > 1: trail += '?' + url.split('?')[1] with open(path + name + trail, 'w+') as out_file: - out_file.write(response.encode('utf-8')) + out_file.write(response.encode('utf-8',errors=encoding_error_response)) diff --git a/core/utils.py b/core/utils.py index 905d390..20483d8 100644 --- a/core/utils.py +++ b/core/utils.py @@ -75,14 +75,14 @@ def remove_regex(urls, regex): return non_matching_urls -def writer(datasets, dataset_names, output_dir): +def writer(datasets, dataset_names, output_dir,encoding_error_response): """Write the results.""" for dataset, dataset_name in zip(datasets, dataset_names): if dataset: filepath = output_dir + '/' + dataset_name + '.txt' with open(filepath, 'w+') as out_file: joined = '\n'.join(dataset) - out_file.write(str(joined.encode('utf-8').decode('utf-8'))) + out_file.write(str(joined.encode('utf-8',errors=encoding_error_response).decode('utf-8'))) out_file.write('\n') @@ -98,12 +98,12 @@ def timer(diff, processed): return minutes, seconds, time_per_request -def entropy(string): +def entropy(string,encoding_error_response): """Calculate the entropy of a string.""" entropy = 0 for number in range(256): - result = float(string.encode('utf-8').count( - chr(number))) / len(string.encode('utf-8')) + result = float(string.encode('utf-8',errors=encoding_error_response).count( + chr(number))) / len(string.encode('utf-8',errors=encoding_error_response)) if result != 0: entropy = entropy - result * math.log(result, 2) return entropy diff --git a/photon.py b/photon.py index 37b7285..3d2d6d4 100644 --- a/photon.py +++ b/photon.py @@ -80,7 +80,7 @@ type=float) parser.add_argument('-p', '--proxy', help='Proxy server IP:PORT or DOMAIN:PORT', dest='proxies', type=proxy_type) - +parser.add_argument('--encoding-error', help='encoding error response parameter', dest='encoding_error',default=['strict'],choices=['backslashreplace','ignore','namereplace','strict','replace','xmlcharrefreplace',]) # Switches parser.add_argument('--clone', help='clone the website locally', dest='clone', action='store_true') @@ -190,6 +190,7 @@ host = urlparse(main_url).netloc output_dir = args.output or host +encoding_error_response = args.encoding_error try: domain = top_level(main_url) @@ -240,7 +241,7 @@ def extractor(url): """Extract details from the response body.""" response = requester(url, main_url, delay, cook, headers, timeout, host, proxies, user_agents, failed, processed) if clone: - mirror(url, response) + mirror(url, response,encoding_error_response) matches = rhref.findall(response) for link in matches: # Remove everything after a "#" to deal with in-page anchors @@ -282,7 +283,7 @@ def extractor(url): if api: matches = rentropy.findall(response) for match in matches: - if entropy(match) >= 4: + if entropy(match,encoding_error_response) >= 4: verb('Key', match) keys.add(url + ': ' + match) @@ -382,7 +383,7 @@ def jscanner(url): dataset_names = ['files', 'intel', 'robots', 'custom', 'failed', 'internal', 'scripts', 'external', 'fuzzable', 'endpoints', 'keys'] -writer(datasets, dataset_names, output_dir) +writer(datasets, dataset_names, output_dir,encoding_error_response) # Printing out results print(('%s-%s' % (red, end)) * 50) for dataset, dataset_name in zip(datasets, dataset_names): @@ -407,7 +408,7 @@ def jscanner(url): from plugins.find_subdomains import find_subdomains subdomains = find_subdomains(domain) print('%s %i subdomains found' % (info, len(subdomains))) - writer([subdomains], ['subdomains'], output_dir) + writer([subdomains], ['subdomains'], output_dir,encoding_error_response) datasets['subdomains'] = subdomains from plugins.dnsdumpster import dnsdumpster print('%s Generating DNS map' % run) From 847fcad8d8736bc2f6a9bb8e70ec4336a99dc2ca Mon Sep 17 00:00:00 2001 From: ege Date: Sun, 7 Mar 2021 21:09:00 +0300 Subject: [PATCH 2/2] added parameter; encoding error response --- photon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/photon.py b/photon.py index 3d2d6d4..ed45946 100644 --- a/photon.py +++ b/photon.py @@ -80,7 +80,7 @@ type=float) parser.add_argument('-p', '--proxy', help='Proxy server IP:PORT or DOMAIN:PORT', dest='proxies', type=proxy_type) -parser.add_argument('--encoding-error', help='encoding error response parameter', dest='encoding_error',default=['strict'],choices=['backslashreplace','ignore','namereplace','strict','replace','xmlcharrefreplace',]) +parser.add_argument('--encoding-error', help='encoding error response parameter', dest='encoding_error',default='strict',choices=['backslashreplace','ignore','namereplace','strict','replace','xmlcharrefreplace',]) # Switches parser.add_argument('--clone', help='clone the website locally', dest='clone', action='store_true')