From c36a14a65e1bf7f95bc516ac532eeb5a30947622 Mon Sep 17 00:00:00 2001 From: Osaetin Daniel Date: Mon, 5 Feb 2018 20:45:42 +0100 Subject: [PATCH 1/5] Started work on django plugins --- staticfy/plugins/django/__init__.py | 3 ++ staticfy/plugins/django/django.py | 31 +++++++++++++++++++++ staticfy/plugins/django/tests.py | 19 +++++++++++++ staticfy/staticfy.py | 43 ++++++++++++++++------------- tests.py | 1 + 5 files changed, 78 insertions(+), 19 deletions(-) create mode 100644 staticfy/plugins/django/__init__.py create mode 100644 staticfy/plugins/django/django.py create mode 100644 staticfy/plugins/django/tests.py diff --git a/staticfy/plugins/django/__init__.py b/staticfy/plugins/django/__init__.py new file mode 100644 index 0000000..74d5976 --- /dev/null +++ b/staticfy/plugins/django/__init__.py @@ -0,0 +1,3 @@ +"""django plugin.""" + +from .django import transform diff --git a/staticfy/plugins/django/django.py b/staticfy/plugins/django/django.py new file mode 100644 index 0000000..576afcc --- /dev/null +++ b/staticfy/plugins/django/django.py @@ -0,0 +1,31 @@ +"""Django plugin for django templates.""" + +import re + + +def transform_extends_tag(html): + """ + Convert to {% extends ".*" %}. + + Also remove the closing tag + """ + opening_regex = r'' + closing_regex = r'' + + html = re.sub(opening_regex, r'{% extends "\1" %}', html) + html = re.sub(closing_regex, '', html) + + return html + + +def transform(file): + """Transform posthtml to django templates.""" + result = [] + + for line in file: + + line = transform_extends_tag(line) + + result.append(line) + + return result diff --git a/staticfy/plugins/django/tests.py b/staticfy/plugins/django/tests.py new file mode 100644 index 0000000..0e85c5f --- /dev/null +++ b/staticfy/plugins/django/tests.py @@ -0,0 +1,19 @@ +"""Tests for the django plugin.""" + +import unittest + + +class DjangoTestCase(unittest.TestCase): + """Tests for the django plugin.""" + + def test_extends_tag(self): + pass + + def test_include_tag(self): + pass + + def test_block_tag(self): + pass + + def test_include_static_tag(self): + pass \ No newline at end of file diff --git a/staticfy/staticfy.py b/staticfy/staticfy.py index 5507610..8dae627 100755 --- a/staticfy/staticfy.py +++ b/staticfy/staticfy.py @@ -1,13 +1,16 @@ """Staticfy.py.""" -from bs4 import BeautifulSoup import sys import re import os import errno import argparse import json -from .config import frameworks + +from bs4 import BeautifulSoup + +from plugins.django import transform as django +from config import frameworks def makedir(path): @@ -72,8 +75,11 @@ def get_elements(html_file, tags): Returns a list of tuples with the attribute as first item and the list of elements as the second item. """ - with open(html_file) as f: - document = BeautifulSoup(f, 'html.parser') + with open(html_file) as file: + + file = django(file) + + document = BeautifulSoup(''.join(file), 'html.parser') def condition(tag, attr): # Don't include external links @@ -83,29 +89,28 @@ def condition(tag, attr): all_tags = [(attr, document.find_all(condition(tag, attr))) for tag, attr in tags] - return all_tags + return all_tags, file def replace_lines(html_file, transformed): """Replace lines in the old file with the transformed lines.""" result = [] - with open(html_file, 'r') as input_file: - for line in input_file: - # replace all single quotes with double quotes - line = re.sub(r'\'', '"', line) + for line in html_file: + # replace all single quotes with double quotes + line = re.sub(r'\'', '"', line) - for attr, value, new_link in transformed: - if attr in line and value in line: + for attr, value, new_link in transformed: + if attr in line and value in line: - # replace old link with new staticfied link - new_line = line.replace(value, new_link) + # replace old link with new staticfied link + new_line = line.replace(value, new_link) - result.append(new_line) - break - else: - result.append(line) + result.append(new_line) + break + else: + result.append(line) - return ''.join(result) + return ''.join(result) def staticfy(html_file, args=argparse.ArgumentParser()): @@ -133,7 +138,7 @@ def staticfy(html_file, args=argparse.ArgumentParser()): tags = tags - exc_tags # get elements we're interested in - matches = get_elements(html_file, tags) + matches, html_file = get_elements(html_file, tags) # transform old links to new links transformed = transform(matches, framework, namespace, static_endpoint) diff --git a/tests.py b/tests.py index 835cb06..c8b5e60 100644 --- a/tests.py +++ b/tests.py @@ -5,6 +5,7 @@ import os from staticfy.staticfy import staticfy + class DummyParser(): """ Empty class that simulates the parser in the main program. From cf26f8615279adae7574e61bbeb59d6905a00830 Mon Sep 17 00:00:00 2001 From: Osaetin Daniel Date: Tue, 6 Feb 2018 17:22:18 +0100 Subject: [PATCH 2/5] Added transforms for extends, include and block tags --- bin/staticfy | 4 +- staticfy/__init__.py | 3 + staticfy/plugins/__init__.py | 0 staticfy/plugins/django/__init__.py | 3 - staticfy/plugins/django/django.py | 31 ---------- staticfy/plugins/django/tests.py | 19 ------- staticfy/plugins/django_posthtml/__init__.py | 3 + staticfy/plugins/django_posthtml/posthtml.py | 57 +++++++++++++++++++ .../plugins/django_posthtml/test_posthtml.py | 54 ++++++++++++++++++ staticfy/staticfy.py | 9 +-- tests.py | 2 +- 11 files changed, 125 insertions(+), 60 deletions(-) create mode 100644 staticfy/plugins/__init__.py delete mode 100644 staticfy/plugins/django/__init__.py delete mode 100644 staticfy/plugins/django/django.py delete mode 100644 staticfy/plugins/django/tests.py create mode 100644 staticfy/plugins/django_posthtml/__init__.py create mode 100644 staticfy/plugins/django_posthtml/posthtml.py create mode 100644 staticfy/plugins/django_posthtml/test_posthtml.py diff --git a/bin/staticfy b/bin/staticfy index b7b4683..e3f338a 100755 --- a/bin/staticfy +++ b/bin/staticfy @@ -1,5 +1,5 @@ -#!/usr/bin/env/ python +#! /usr/bin/env/ python -from staticfy.staticfy import main +from staticfy import main main() diff --git a/staticfy/__init__.py b/staticfy/__init__.py index e69de29..34fed75 100644 --- a/staticfy/__init__.py +++ b/staticfy/__init__.py @@ -0,0 +1,3 @@ +"""Initialize functions""" + +from .staticfy import staticfy, main diff --git a/staticfy/plugins/__init__.py b/staticfy/plugins/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/staticfy/plugins/django/__init__.py b/staticfy/plugins/django/__init__.py deleted file mode 100644 index 74d5976..0000000 --- a/staticfy/plugins/django/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""django plugin.""" - -from .django import transform diff --git a/staticfy/plugins/django/django.py b/staticfy/plugins/django/django.py deleted file mode 100644 index 576afcc..0000000 --- a/staticfy/plugins/django/django.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Django plugin for django templates.""" - -import re - - -def transform_extends_tag(html): - """ - Convert to {% extends ".*" %}. - - Also remove the closing tag - """ - opening_regex = r'' - closing_regex = r'' - - html = re.sub(opening_regex, r'{% extends "\1" %}', html) - html = re.sub(closing_regex, '', html) - - return html - - -def transform(file): - """Transform posthtml to django templates.""" - result = [] - - for line in file: - - line = transform_extends_tag(line) - - result.append(line) - - return result diff --git a/staticfy/plugins/django/tests.py b/staticfy/plugins/django/tests.py deleted file mode 100644 index 0e85c5f..0000000 --- a/staticfy/plugins/django/tests.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Tests for the django plugin.""" - -import unittest - - -class DjangoTestCase(unittest.TestCase): - """Tests for the django plugin.""" - - def test_extends_tag(self): - pass - - def test_include_tag(self): - pass - - def test_block_tag(self): - pass - - def test_include_static_tag(self): - pass \ No newline at end of file diff --git a/staticfy/plugins/django_posthtml/__init__.py b/staticfy/plugins/django_posthtml/__init__.py new file mode 100644 index 0000000..6ce16c0 --- /dev/null +++ b/staticfy/plugins/django_posthtml/__init__.py @@ -0,0 +1,3 @@ +"""django plugin.""" + +from .posthtml import transform diff --git a/staticfy/plugins/django_posthtml/posthtml.py b/staticfy/plugins/django_posthtml/posthtml.py new file mode 100644 index 0000000..a51bda3 --- /dev/null +++ b/staticfy/plugins/django_posthtml/posthtml.py @@ -0,0 +1,57 @@ +"""Django plugin for django templates.""" + +import re + + +def transform_extends_tag(html): + """ + Convert to {% extends ".*" %}. + + Also remove the closing tag + """ + opening_regex = r'' + closing_regex = r'' + + html = re.sub(opening_regex, r'{% extends "\1" %}', html) + html = re.sub(closing_regex, '', html) + + return html + + +def transform_include_tag(html): + """Convert to {% include ".*" %}.""" + include_regex = r'' + + html = re.sub(include_regex, r'{% include "\2" %}', html) + + return html + + +def transform_block_tag(html): + """ + Convert to {% extends ".*" %}. + + Also convert the closing to {% endblock %} + """ + opening_regex = r'' + closing_regex = r'' + + html = re.sub(opening_regex, r'{% block \1 %}', html) + html = re.sub(closing_regex, '{% endblock %}', html) + + return html + + +def transform(file): + """Transform posthtml to django templates.""" + result = [] + + for line in file: + # Apply transforms + line = transform_extends_tag(line) + line = transform_include_tag(line) + line = transform_block_tag(line) + + result.append(line) + + return result diff --git a/staticfy/plugins/django_posthtml/test_posthtml.py b/staticfy/plugins/django_posthtml/test_posthtml.py new file mode 100644 index 0000000..c076792 --- /dev/null +++ b/staticfy/plugins/django_posthtml/test_posthtml.py @@ -0,0 +1,54 @@ +"""Tests for the django plugin.""" + +import unittest + +from .posthtml import transform + + +class DjangoTestCase(unittest.TestCase): + """Tests for the django plugin.""" + + def test_extends_tag(self): + """ + The tag should be converted to {% extends ".*" %}. + + The closing tag should also be removed. + """ + result = transform( + ['', '

Hello world

', '
'] + ) + + expected = ['{% extends "base.html" %}', '

Hello world

', ''] + + self.assertEqual(result, expected) + + def test_include_tag(self): + """The tag should be converted to {% include ".*" %}.""" + result = transform( + ['', '

Hello world

'] + ) + + expected = ['{% include "base.html" %}', '

Hello world

'] + + self.assertEqual(result, expected) + + def test_block_tag(self): + """ + The tag should be converted to {% block .* %}. + + The closing tag should be converted to {% endblock %} + """ + result = transform( + ['', '

Hello world

', '
'] + ) + + expected = ['{% block content %}', '

Hello world

', '{% endblock %}'] + + self.assertEqual(result, expected) + + def test_include_static_tag(self): + pass + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/staticfy/staticfy.py b/staticfy/staticfy.py index 8dae627..0efb2e7 100755 --- a/staticfy/staticfy.py +++ b/staticfy/staticfy.py @@ -9,8 +9,8 @@ from bs4 import BeautifulSoup -from plugins.django import transform as django -from config import frameworks +from .plugins.django_posthtml import transform as django_posthtml +from .config import frameworks def makedir(path): @@ -35,8 +35,9 @@ def get_asset_location(element, attr): asset_location = re.match(r'^/?(static)?/?(.*)', element[attr], re.IGNORECASE) - # replace relative links i.e (../../static) + # replace relative links i.e (../../static) or (./) asset_location = asset_location.group(2).replace('../', '') + asset_location = asset_location.replace('./', '') return asset_location @@ -77,7 +78,7 @@ def get_elements(html_file, tags): """ with open(html_file) as file: - file = django(file) + file = django_posthtml(file) document = BeautifulSoup(''.join(file), 'html.parser') diff --git a/tests.py b/tests.py index c8b5e60..1f4cc89 100644 --- a/tests.py +++ b/tests.py @@ -3,7 +3,7 @@ import unittest import os -from staticfy.staticfy import staticfy +from staticfy import staticfy class DummyParser(): From ce4fbdcb74a5b35418809124740c65c32ad3d5a8 Mon Sep 17 00:00:00 2001 From: Osaetin Daniel Date: Tue, 6 Feb 2018 17:24:47 +0100 Subject: [PATCH 3/5] Changed Travis CI to disover plugin tests --- .travis.yml | 2 +- README.md | 32 +++++++++++++++++++++++++------- staticfy/staticfy.py | 4 ++-- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8847dc3..cb8d754 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,4 +5,4 @@ python: # command to install dependencies install: "pip install -r requirements.txt" # command to run tests -script: python tests.py +script: python -m unittest diff --git a/README.md b/README.md index dda27c4..79cf2bb 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ ### Status + [![Build Status](https://travis-ci.org/danidee10/Staticfy.svg?branch=master)](https://travis-ci.org/danidee10/Staticfy) [![Code Climate](https://codeclimate.com/github/danidee10/Staticfy/badges/gpa.svg)](https://codeclimate.com/github/danidee10/Staticfy) # Staticfy + You just got a brand new template fron the front-end designer, Everything's going well, Until you realize the amount of time you'll spend manually changing the links in the html templates you until all the static files and assets are properly linked and the file looks exactly like the demo he/she showed you. with Staticfy you can save that time (and some of your hair) by automatically converting the static urls in your template to dynamic url's that wouldn't break if you decide to move your file to another location. @@ -16,14 +18,17 @@ To this: and then your web framework's templating language can resolve the tags to the right url. # Get it in 10 seconds! + It's available as a package on PyPi so you can install it with ```bash pip install staticfy ``` + That's all! Run it straight from the command line with: + ```bash staticfy staticfy.html --static-endpoint=static --add-tags='{"img": "data-url"}' ``` @@ -37,9 +42,11 @@ staticfy staticfy.html -o new.html ``` ### Before Staticfying + ![alt tag](assets/before.png) --------------------------------------------------------------------------------------------------------------------------------- ### After Staticfying + ![alt tag](assets/after.png) Notice how it preserves the font-awesome css link at the top of the file?, external resources (images, scripts, font-awesome, google-fonts, bootstrap files, disqus e.t.c) which aren't hosted locally with your website won't be staticfied. Staticfy also accepts an optional argument `--static-endpoint` in case you're not using the default static endpoint. @@ -47,6 +54,7 @@ Notice how it preserves the font-awesome css link at the top of the file?, exter Staticy also preserves the indentation and formatting of any html file given to it, so your html file(s) are still look the same way and are still readablebe just the way they were before you staticfied them. # Additional tags and attributes + By default staticfy identifies and staticfies the following tags: 1. img tags with src attributes -- `` 2. link tags with rel attributes -- `` @@ -61,9 +69,11 @@ staticfy staticfy.html --add-tags='{"div": "data-src"}' Sure enough it gets staticfied. ### Before staticfying + ![alt tag](assets/before_add_tag.png) ### After staticfying + ![alt tag](assets/after_add_tag.png) You can exclude certain tags you don't want to be staticfied by specifying the `--exc-tags` parameter, like `--add-tags` it expects a valid JSON string. @@ -78,6 +88,7 @@ It should be noted that sub folders containing html files won't be staticfied, o Whenever you run staticfy on a template or on a folder, a staticfy folder is generated in the present working directory and the staticfied file(s) is placed in that folder, you also need to copy the file(s) over to the appropriate directory to overwrite the existing file with the new one. # Namespacing + When your project gets big, It's necessary to namespace static assets to avoid name collision, you can achieve this by passing a string to the `--namespace` argument. The string would automatically be prepended to the url. For example in django ```bash @@ -88,6 +99,7 @@ Would convert `` to this ` 3.3 (mkdir -p in *nix).""" - try: - os.makedirs(path) - except OSError as e: - if e.errno == errno.EEXIST and os.path.isdir(path): - pass - else: - raise - - def get_asset_location(element, attr): """ Get Asset Location. @@ -60,7 +48,7 @@ def transform(matches, framework, namespace, static_endpoint): sub_dict = { 'static_endpoint': static_endpoint, 'namespace': namespace, 'asset_location': asset_location - } + } transformed_string = frameworks[framework] % sub_dict res = (attribute, element[attribute], transformed_string) @@ -69,57 +57,83 @@ def transform(matches, framework, namespace, static_endpoint): return transformed -def get_elements(html_file, tags): +def transform_using_plugins(html_file, plugins): + """ + Transform the html file using specified plugins. + + Save at the end of the transformation + """ + + with open(html_file) as file: + transformed = [line for line in file] + + for plugin_name in plugins: + plugin = import_module('plugins.' + plugin_name) + transformed = plugin.transform(transformed) + + if plugins: + return transformed + + +def get_elements(html, html_file, tags): """ Extract all the elements we're interested in. Returns a list of tuples with the attribute as first item and the list of elements as the second item. """ - with open(html_file) as file: - file = django_posthtml(file) + document = BeautifulSoup(''.join(html), 'html.parser') - document = BeautifulSoup(''.join(file), 'html.parser') + def no_external_links(tag, attr): + """Don't include external links.""" + return lambda x: x.name == tag \ + and not x.get(attr, 'http').startswith(('http', '//')) - def condition(tag, attr): - # Don't include external links - return lambda x: x.name == tag \ - and not x.get(attr, 'http').startswith(('http', '//')) + all_tags = [(attr, document.find_all(no_external_links(tag, attr))) + for tag, attr in tags + ] - all_tags = [(attr, document.find_all(condition(tag, attr))) - for tag, attr in tags] + # Save contents back to file + with open(html_file, 'w') as file: + file.write(''.join(html)) - return all_tags, file + return all_tags def replace_lines(html_file, transformed): """Replace lines in the old file with the transformed lines.""" result = [] - for line in html_file: - # replace all single quotes with double quotes - line = re.sub(r'\'', '"', line) - for attr, value, new_link in transformed: - if attr in line and value in line: + with open(html_file) as file: + for line in file: + # replace all single quotes with double quotes + line = re.sub(r'\'', '"', line) - # replace old link with new staticfied link - new_line = line.replace(value, new_link) + for attr, value, new_link in transformed: + if attr in line and value in line: - result.append(new_line) - break - else: - result.append(line) + # replace old link with new staticfied link + new_line = line.replace(value, new_link) + + result.append(new_line) + break + else: + result.append(line) return ''.join(result) -def staticfy(html_file, args=argparse.ArgumentParser()): +def staticfy(html_file, config=None, args=argparse.ArgumentParser()): """ Staticfy method. Loop through each line of the file and replaces the old links """ + + if not config: + config = {'plugins': ['django_posthtml']} + # unpack arguments static_endpoint = args.static_endpoint or 'static' framework = args.framework or os.getenv('STATICFY_FRAMEWORK', 'flask') @@ -138,8 +152,11 @@ def staticfy(html_file, args=argparse.ArgumentParser()): exc_tags = {(tag, attr) for tag, attr in exc_tags.items()} tags = tags - exc_tags + # apply plugin(s) transformation + html = transform_using_plugins(html_file, config['plugins']) + # get elements we're interested in - matches, html_file = get_elements(html_file, tags) + matches = get_elements(html, html_file, tags) # transform old links to new links transformed = transform(matches, framework, namespace, static_endpoint) @@ -179,7 +196,7 @@ def parse_cmd_arguments(): return args -def main(): +def main(config): """Main method.""" args = parse_cmd_arguments() html_file = args.file @@ -192,9 +209,5 @@ def main(): 'string e.g {}'.format('\'{"img": "data-url"}\'') + '\033[0m') sys.exit(1) - staticfied = staticfy(html_file, args=args) + staticfied = staticfy(html_file, config=config, args=args) file_ops(staticfied, args=args) - - -if __name__ == '__main__': - main() diff --git a/tests.py b/tests.py index 1f4cc89..cd9f18b 100644 --- a/tests.py +++ b/tests.py @@ -158,8 +158,12 @@ def test_replace_relative_links(self): ) self.assertEqual(result, expected_result) - def test_filenotfound_exception(self): - self.assertRaises(IOError, staticfy, 'Invalid html file', args=self.args) + def test_nonexistent_html_file(self): + """ + An IOError should be raised + if we try to staticfy a nonexistent file. + """ + self.assertRaises(IOError, staticfy, 'not_found', args=self.args) @classmethod def tearDownClass(cls):