Add generated unit test coverage for lazy import, part 1 (#303)

The Python 3.8 showcase unit tests (and all 'real' generated unit
tests when using the lazy import feature) include a test to verify
that the logic used for lazy import is covered.

If the classes and submodules are available it is assumed that they
are the correct ones; asserting that there is no mismatch between lazy
import path and corresponding object is outside the current scope of
these tests.

Lazy import is currently only available for the top level of a module and its corresponding types submodule. Other submodules will be converted to use lazy imports as part of subsequent commits.

Author: software-dov

gapic/templates/%namespace/%name_%version/%sub/__init__.py.j2

@@ -30,14 +30,14 @@ def from_snake_case(s):
 
 
 if sys.version_info < (3, 7):
-    raise ImportError('This module requires Python 3.7 or later.')
+    raise ImportError('This module requires Python 3.7 or later.')  # pragma: NO COVER
 
 _lazy_name_to_package_map = {
     'types': '{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.types',
   {%- for service in api.services.values()|sort(attribute='name')|unique(attribute='name') if service.meta.address.subpackage == api.subpackage_view %}
     '{{ service.client_name|snake_case }}': '{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.services.{{ service.name|snake_case }}.client',
-    '{{ service.transport_name|snake_case }}': '{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.services.transports.base',
-    '{{ service.grpc_transport_name|snake_case }}': '{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.services.transports.grpc',
+    '{{ service.transport_name|snake_case }}': '{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.services.{{ service.name|snake_case }}.transports.base',
+    '{{ service.grpc_transport_name|snake_case }}': '{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.services.{{ service.name|snake_case }}.transports.grpc',
   {%- endfor %}
 }
 
@@ -56,8 +56,9 @@ def __getattr__(name):  # Requires Python >= 3.7
     if name == '__all__':
         all_names = globals()['__all__'] = sorted(
             chain(
-                (from_snake_case(k) for k in _lazy_name_to_package_map),
+                (from_snake_case(k) for k in _lazy_name_to_package_map if k != 'types'),
                 _lazy_type_to_package_map,
+                ['types'],
             )
         )
         return all_names

gapic/templates/%namespace/%name_%version/%sub/types/__init__.py.j2

@@ -1,13 +1,53 @@
 {% extends '_base.py.j2' %}
 
 {% block content %}
+{% if opts.lazy_import -%} {# lazy import #}
+import importlib
+import sys
+
+
+if sys.version_info < (3, 7):
+    raise ImportError('This module requires Python 3.7 or later.')  # pragma: NO COVER
+
+
+_lazy_type_to_package_map = {
+  {%- filter sort_lines %}
+{%- for proto in api.protos.values() if proto.meta.address.subpackage == api.subpackage_view %}{%- for message in proto.messages.values() %}
+    '{{ message.name }}': '{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.types.{{ proto.module_name }}',
+{%- endfor %}
+{%- for enum in proto.enums.values() %}
+    '{{ enum.name }}': '{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.types.{{ proto.module_name }}',
+{%- endfor %}{%- endfor %}{%- endfilter %}
+}
+
+
+# Background on how this behaves: https://www.python.org/dev/peps/pep-0562/
+def __getattr__(name):  # Requires Python >= 3.7
+    if name == '__all__':
+        all_names = globals()['__all__'] = sorted(_lazy_type_to_package_map)
+        return all_names
+    elif name in _lazy_type_to_package_map:
+        module = importlib.import_module(f'{_lazy_type_to_package_map[name]}')
+        klass = getattr(module, name)
+        {# new_klass = type(name, (klass,), {'__doc__': klass.__doc__}) #}
+        globals()[name] = klass
+        return klass
+    else:
+        raise AttributeError(f'unknown sub-module {name!r}.')
+
+
+def __dir__():
+    return globals().get('__all__') or __getattr__('__all__')
+
+{% else -%}
 {% for p in api.protos.values() if p.file_to_generate and p.messages -%}
-from .{{ p.module_name }} import ({% for m in p.messages.values() %}{{ m.name }}, {% endfor %})
+from .{{p.module_name }} import ({% for m in p.messages.values() %}{{ m.name }}, {% endfor %})
 {% endfor %}
 
 __all__ = (
 {%- for p in api.protos.values() if p.file_to_generate %}{% for m in p.messages.values() %}
     '{{ m.name }}',
 {%- endfor %}{% endfor %}
 )
+{% endif -%} {# lazy import #}
 {% endblock %}

gapic/templates/tests/unit/%name_%version/%sub/test_%service.py.j2

@@ -312,6 +312,75 @@ def test_{{ method.name|snake_case }}_raw_page_lro():
 
 {% endfor -%} {#- method in methods #}
 
+{% if opts.lazy_import -%} {# lazy import #}
+def test_module_level_imports():
+    # Use the other transport import path so that code gets tested.
+    from {{ (api.naming.module_namespace + (api.naming.versioned_module_name,) + service.meta.address.subpackage)|join(".") }} import {{ service.name }}GrpcTransport
+    transport = {{ service.name }}GrpcTransport(
+        credentials=credentials.AnonymousCredentials(),
+    )
+
+    from {{ (api.naming.module_namespace + (api.naming.versioned_module_name,) + service.meta.address.subpackage)|join(".") }} import {{ service.client_name }}
+    client = {{ service.client_name }}(transport=transport)
+    assert client._transport is transport
+
+    from {{ (api.naming.module_namespace + (api.naming.versioned_module_name,) + service.meta.address.subpackage)|join(".") }} import {{ service.name|snake_case }}_grpc_transport
+    transport2 = {{ service.name|snake_case }}_grpc_transport.{{ service.name }}GrpcTransport(
+        credentials=credentials.AnonymousCredentials(),
+    )
+
+    client2 = {{ service.client_name }}(transport=transport2)
+    assert client2._transport is transport2
+
+    {% with type_name = cycler(*service.methods.values()).next().input.name -%}
+    from {{ (api.naming.module_namespace + (api.naming.versioned_module_name,) + service.meta.address.subpackage)|join(".") }} import {{ type_name }}
+    type_ = {{ type_name }}()
+
+    try:
+        from {{ (api.naming.module_namespace + (api.naming.versioned_module_name,) + service.meta.address.subpackage)|join(".") }} import {{ type_name|lower }}_squidification
+    except (AttributeError, ImportError) as e:
+        pass
+    else:
+        assert False
+    {% endwith -%}
+
+    import {{ (api.naming.module_namespace + (api.naming.versioned_module_name,) + service.meta.address.subpackage)|join(".") }} as mod
+    all_names = dir(mod)
+    expected_names = sorted([
+        'types',
+        {%- for service in api.services.values()|sort(attribute='name')|unique(attribute='name') if service.meta.address.subpackage == api.subpackage_view %}
+        '{{ service.client_name }}',
+        '{{ service.transport_name }}',
+        '{{ service.grpc_transport_name }}',
+        {%- endfor %}
+        {%- for proto in api.protos.values() if proto.meta.address.subpackage == api.subpackage_view %}{%- for message in proto.messages.values() %}
+        '{{ message.name }}',
+        {%- endfor %}
+        {%- for enum in proto.enums.values() %}
+        '{{ enum.name }}'
+        {% endfor %}{%- endfor %}
+    ])
+    assert all_names == expected_names
+
+    {% with type_name = cycler(*service.methods.values()).next().input.name -%}
+    from {{ (api.naming.module_namespace + (api.naming.versioned_module_name,) + service.meta.address.subpackage)|join(".") }}.types import {{ type_name }}
+    type_ = {{ type_name }}()
+    {% endwith -%}
+
+    import {{ (api.naming.module_namespace + (api.naming.versioned_module_name,) + service.meta.address.subpackage)|join(".") }}.types as types
+    all_types = dir(types)
+    expected_types = sorted([
+    {%- for proto in api.protos.values() if proto.meta.address.subpackage == api.subpackage_view %}{%- for message in proto.messages.values() %}
+        '{{ message.name }}',
+        {%- endfor %}
+        {%- for enum in proto.enums.values() %}
+        '{{ enum.name }}',
+        {% endfor %}{%- endfor %}
+    ])
+    assert all_types == expected_types
+
+{% endif -%} {# lazy import #}
+
 def test_credentials_transport_error():
     # It is an error to provide credentials and a transport instance.
     transport = transports.{{ service.name }}GrpcTransport(

noxfile.py

@@ -109,15 +109,22 @@ def showcase_unit(session):
         )
 
         # Write out a client library for Showcase.
-        session.run('protoc',
-                    f'--descriptor_set_in={tmp_dir}{os.path.sep}showcase.desc',
-                    f'--python_gapic_out={tmp_dir}',
-                    'google/showcase/v1beta1/echo.proto',
-                    'google/showcase/v1beta1/identity.proto',
-                    'google/showcase/v1beta1/messaging.proto',
-                    'google/showcase/v1beta1/testing.proto',
-                    external=True,
-                    )
+        args = [
+            'protoc',
+            f'--descriptor_set_in={tmp_dir}{os.path.sep}showcase.desc',
+            f'--python_gapic_out={tmp_dir}',
+            'google/showcase/v1beta1/echo.proto',
+            'google/showcase/v1beta1/identity.proto',
+            'google/showcase/v1beta1/messaging.proto',
+            'google/showcase/v1beta1/testing.proto',
+        ]
+        if session.python == '3.8':
+            args.append('--python_gapic_opt=lazy-import')
+
+        session.run(
+            *args,
+            external=True,
+        )
 
         # Install the library.
         session.chdir(tmp_dir)