Merge pull request #538 from troyready/fix_yaml_template_load

fix loading of yaml templates with CFN tags

Author: ejholmes

stacker/actions/diff.py

@@ -4,11 +4,10 @@
 import sys
 from operator import attrgetter
 
-import yaml
-
 from .base import plan, build_walker
 from . import build
 from .. import exceptions
+from ..util import parse_cloudformation_template
 from ..status import NotSubmittedStatus, NotUpdatedStatus, COMPLETE
 
 logger = logging.getLogger(__name__)
@@ -228,13 +227,13 @@ def _diff_stack(self, stack, **kwargs):
             self._print_new_stack(new_stack, parameters)
         else:
             # Diff our old & new stack/parameters
-            old_template = yaml.load(old_template)
-            if isinstance(old_template, str):
+            old_template = parse_cloudformation_template(old_template)
+            if isinstance(old_template, (str, unicode)):
                 # YAML templates returned from CFN need parsing again
                 # "AWSTemplateFormatVersion: \"2010-09-09\"\nParam..."
                 # ->
                 # AWSTemplateFormatVersion: "2010-09-09"
-                old_template = yaml.load(old_template)
+                old_template = parse_cloudformation_template(old_template)
             old_stack = self._normalize_json(
                 json.dumps(old_template,
                            sort_keys=True,

stacker/awscli_yamlhelper.py

@@ -0,0 +1,76 @@
+# Copyright 2012-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+import json
+import yaml
+from yaml.resolver import ScalarNode, SequenceNode
+
+from botocore.compat import six
+
+
+def intrinsics_multi_constructor(loader, tag_prefix, node):
+    """
+    YAML constructor to parse CloudFormation intrinsics.
+    This will return a dictionary with key being the instrinsic name
+    """
+
+    # Get the actual tag name excluding the first exclamation
+    tag = node.tag[1:]
+
+    # Some intrinsic functions doesn't support prefix "Fn::"
+    prefix = "Fn::"
+    if tag in ["Ref", "Condition"]:
+        prefix = ""
+
+    cfntag = prefix + tag
+
+    if tag == "GetAtt" and isinstance(node.value, six.string_types):
+        # ShortHand notation for !GetAtt accepts Resource.Attribute format
+        # while the standard notation is to use an array
+        # [Resource, Attribute]. Convert shorthand to standard format
+        value = node.value.split(".", 1)
+
+    elif isinstance(node, ScalarNode):
+        # Value of this node is scalar
+        value = loader.construct_scalar(node)
+
+    elif isinstance(node, SequenceNode):
+        # Value of this node is an array (Ex: [1,2])
+        value = loader.construct_sequence(node)
+
+    else:
+        # Value of this node is an mapping (ex: {foo: bar})
+        value = loader.construct_mapping(node)
+
+    return {cfntag: value}
+
+
+def yaml_dump(dict_to_dump):
+    """
+    Dumps the dictionary as a YAML document
+    :param dict_to_dump:
+    :return:
+    """
+    return yaml.safe_dump(dict_to_dump, default_flow_style=False)
+
+
+def yaml_parse(yamlstr):
+    """Parse a yaml string"""
+    try:
+        # PyYAML doesn't support json as well as it should, so if the input
+        # is actually just json it is better to parse it with the standard
+        # json parser.
+        return json.loads(yamlstr)
+    except ValueError:
+        yaml.SafeLoader.add_multi_constructor(
+            "!", intrinsics_multi_constructor)
+        return yaml.safe_load(yamlstr)

stacker/blueprints/raw.py

@@ -3,8 +3,7 @@
 import hashlib
 import json
 
-import yaml
-
+from ..util import parse_cloudformation_template
 from ..exceptions import MissingVariable, UnresolvedVariable
 
 
@@ -101,7 +100,7 @@ def to_dict(self):
             dict: the loaded template as a python dictionary
 
         """
-        return yaml.load(self.rendered)
+        return parse_cloudformation_template(self.rendered)
 
     def render_template(self):
         """Load template and generate its md5 hash."""

stacker/tests/fixtures/cfn_template.yaml

@@ -7,6 +7,14 @@ Parameters:
     Default: default
     Type: CommaDelimitedList
 Resources:
+  Bucket:
+    Type: AWS::S3::Bucket
+    Properties:
+      BucketName:
+        !Join
+          - "-"
+          - - !Ref "AWS::StackName"
+            - !Ref "AWS::Region"
   Dummy:
     Type: AWS::CloudFormation::WaitConditionHandle
 Outputs:

stacker/tests/test_util.py

@@ -19,6 +19,7 @@
     get_client_region,
     get_s3_endpoint,
     s3_bucket_location_constraint,
+    parse_cloudformation_template,
     Extractor,
     TarExtractor,
     TarGzipExtractor,
@@ -144,6 +145,45 @@ def test_s3_bucket_location_constraint(self):
                 result
             )
 
+    def test_parse_cloudformation_template(self):
+        template = """AWSTemplateFormatVersion: "2010-09-09"
+Parameters:
+  Param1:
+    Type: String
+Resources:
+  Bucket:
+    Type: AWS::S3::Bucket
+    Properties:
+      BucketName:
+        !Join
+          - "-"
+          - - !Ref "AWS::StackName"
+            - !Ref "AWS::Region"
+Outputs:
+  DummyId:
+    Value: dummy-1234"""
+        parsed_template = {
+            'AWSTemplateFormatVersion': '2010-09-09',
+            'Outputs': {'DummyId': {'Value': 'dummy-1234'}},
+            'Parameters': {'Param1': {'Type': 'String'}},
+            'Resources': {
+                'Bucket': {'Type': 'AWS::S3::Bucket',
+                           'Properties': {
+                               'BucketName': {
+                                   u'Fn::Join': [
+                                       '-',
+                                       [{u'Ref': u'AWS::StackName'},
+                                        {u'Ref': u'AWS::Region'}]
+                                   ]
+                               }
+                           }}
+            }
+        }
+        self.assertEqual(
+            parse_cloudformation_template(template),
+            parsed_template
+        )
+
     def test_extractors(self):
         self.assertEqual(Extractor('test.zip').archive, 'test.zip')
         self.assertEqual(TarExtractor().extension(), '.tar')

stacker/util.py

@@ -21,6 +21,7 @@
 from yaml.constructor import ConstructorError
 from yaml.nodes import MappingNode
 
+from .awscli_yamlhelper import yaml_parse
 from stacker.session_cache import get_session
 
 logger = logging.getLogger(__name__)
@@ -496,6 +497,17 @@ def ensure_s3_bucket(s3_client, bucket_name, bucket_region):
             raise
 
 
+def parse_cloudformation_template(template):
+    """Parse CFN template string.
+
+    Leverages the vendored aws-cli yamlhelper to handle JSON or YAML templates.
+
+    Args:
+        template (str): The template body.
+    """
+    return yaml_parse(template)
+
+
 class Extractor(object):
     """Base class for extractors."""