Merge pull request #20 from Yancey1989/fluid_ci

Enable Python linters on CI

Author: wangkuiyi

.pre-commit-config.yaml

@@ -0,0 +1,9 @@
+repos:
+- repo: local
+  hooks:
+  - id: pylint
+    name: pylint
+    entry: bash ./scripts/pylint.sh
+    language: system
+    files: \.py$
+    verbose: true

.travis.yml

@@ -0,0 +1,22 @@
+dist: xenial
+cache: pip
+branches:
+  only:
+  - master
+  - develop
+  # https://docs.travis-ci.com/user/customizing-the-build/#safelisting-or-blocklisting-branches
+  # safe list can prevent tag building, add rexp to detect tag
+  - "/^v\\d+\\.\\d+(\\.\\d+)?(-\\S*)?$/"
+
+language: python
+python:
+- 3.6
+
+install:
+- pip install -r requirements.txt
+- python setup.py bdist_wheel
+- pip install dist/*whl
+
+script:
+- bash ./scripts/pylint.sh
+- pytest .

fluid/__init__.py

@@ -1 +1,10 @@
-from fluid.pyfunc import *
+# -*- coding: utf-8 -*-
+'''This module expose Fluid API for users to write Fluid program
+'''
+
+from . import pyfunc
+from .pyfunc import * # noqa
+from . import version
+from .version import __version__ # noqa
+
+__all__ = pyfunc.__all__ + version.__all__

fluid/pyfunc.py

@@ -13,6 +13,11 @@
 import fluid.tekton as tekton
 import fluid.k8s as k8s
 
+__all__ = [
+    'Secret', 'FluidSyntaxError', 'dump_yaml', 'task', 'step', 'git_resource',
+    'image_resource', 'service_account'
+]
+
 
 def dump_yaml(content):
     '''Pretty print a Python dictionary as a YAML'''
@@ -35,7 +40,9 @@ class FluidSyntaxError(Exception):
 
 
 def _resources_before_params(func):
-    '''Check annotated params (resources) are all before non-annoted params'''
+    '''Check annotated params (resources)
+       are all before non-annoted params
+    '''
     seen_param = False
     argspec = inspect.getfullargspec(func)
     for i, arg in enumerate(argspec.args):
@@ -46,7 +53,8 @@ def _resources_before_params(func):
             _resource_has_no_default(i, arg, anno, argspec)
             _resources_annotation_io_type(arg, anno)
             if seen_param:
-                raise FluidSyntaxError(f"{arg} is annotated and is after a non-annotated param")
+                raise FluidSyntaxError(f"{arg} is annotated and is \
+                    after a non-annotated param")
 
 
 def _resource_has_no_default(i, arg, anno, argspec):

fluid/tekton.py

@@ -39,7 +39,8 @@ def task_params(argspec):
         if argspec.annotations.get(arg) is None:  # is param
             _ps.append(task_param(
                 arg,
-                None if i < first_default else argspec.defaults[i-first_default]))
+                None if i < first_default else argspec.defaults[
+                    i-first_default]))
     return _ps
 
 
@@ -107,7 +108,7 @@ def task_run(func, args):
             "outputs": {
                 "resources": _or,
             }})
-    global SERVICE_ACCOUNT_NAME
+    global SERVICE_ACCOUNT_NAME  # pylint: disable=global-statement
     if SERVICE_ACCOUNT_NAME is not None:
         _r["spec"]["serviceAccountName"] = SERVICE_ACCOUNT_NAME
     return _r
@@ -161,23 +162,24 @@ def task_run_resource(param, arg):
         }}
 
 
-class FakeGitResource:
+class FakeGitResource:  # pylint: disable=too-few-public-methods
     '''Used in dry-run a Task function, which might call res_param.revision'''
 
     def __init__(self, io, arg):
         self.url = f"$({io}s.resources.{k8s.safe_name(arg)}.url)"
         self.revision = f"$({io}s.resources.{k8s.safe_name(arg)}.revision)"
 
 
-class FakeImageResource:
+class FakeImageResource:    # pylint: disable=too-few-public-methods
     '''Used in dry-run a Task function, which might call res_param.url'''
 
     def __init__(self, io, arg):
         self.url = f"$({io}s.resources.{k8s.safe_name(arg)}.url)"
 
 
 def _fake_resource(_io, _typ, arg):
-    return FakeGitResource(_io, arg) if _typ == "git" else FakeImageResource(_io, arg)
+    return FakeGitResource(_io, arg) \
+        if _typ == "git" else FakeImageResource(_io, arg)
 
 
 STEPS = []  # For holding steps of a Task.
@@ -194,7 +196,7 @@ def task_steps(func):
         else:
             fake_args.append(f"$(inputs.params.{k8s.safe_name(arg)})")
 
-    global STEPS
+    global STEPS    # pylint: disable=global-statement
     STEPS = []
     func(*fake_args)
     return STEPS
@@ -210,12 +212,14 @@ def add_step(name, image, cmd, args, env):
     }
     if env is not None:
         _s["env"] = step_env(env)
-    global STEPS
+    global STEPS     # pylint: disable=global-statement
     STEPS.append(_s)
 
 
 def step_env(env):
-    '''For the convenience of Fluid users, env is a dict; Tekton requires a list'''
+    '''For the convenience of Fluid users, env is a dict;
+       Tekton requires a list
+    '''
     return [{"name": k, "value": v} for i, (k, v) in enumerate(env.items())]
 
 
@@ -229,7 +233,8 @@ def _resource(name, typ, params):
         },
         "spec": {
             "type": typ,
-            "params": [{"name": k, "value": v} for i, (k, v) in enumerate(params.items())]}}
+            "params": [{"name": k, "value": v} for i, (k, v) in enumerate(
+                params.items())]}}
 
 
 def git_resource(name, url, revision):

fluid/version.py

@@ -1,4 +1,8 @@
 # -*- coding: utf-8 -*-
+'''This module version presentents Fluid version
+'''
+
+__all__ = ['__version__']
 
 VERSION = (0, 0, 1, 'dev')
 

requirements.txt

@@ -0,0 +1,3 @@
+pylint
+flake8
+pytest

scripts/pylint.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+changed_py_files=$(git diff --cached --name-only --diff-filter=ACMR | grep '\.py$')
+echo $changed_py_files
+if [[ $changed_py_files == "" ]]; then
+    exit 0
+fi
+
+pylint $changed_py_files
+flake8 $changed_py_files

setup.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-
+'''This module packages and distributes Fluid'''
 import io
 import os
 
@@ -15,8 +15,8 @@
 VERSION = None
 
 # What packages are required for this module to be executed?
-SETUP_REQUIRED = ['pytest-runner']
-TEST_REQUIRED = [
+REQUIRES = ['pyyaml']
+TEST_REQUIRED = REQUIRES + [
     'pytest',
 ]
 
@@ -26,39 +26,40 @@
 # The rest you shouldn't have to touch too much :)
 # ------------------------------------------------
 # Except, perhaps the License and Trove Classifiers!
-# If you do change the License, remember to change the Trove Classifier for that!
+# If you do change the License, remember to change the Trove Classifier
+# for that!
 
-here = os.path.abspath(os.path.dirname(__file__))
+HERE = os.path.abspath(os.path.dirname(__file__))
 
 # Import the README and use it as the long-description.
 # Note: this will only work if 'README.md' is present in your MANIFEST.in file!
 try:
-    with io.open(os.path.join(here, 'README.md'), encoding='utf-8') as f:
-        long_description = '\n' + f.read()
+    with io.open(os.path.join(HERE, 'README.md'), encoding='utf-8') as f:
+        LONG_DESCRIPTION = '\n' + f.read()
 except FileNotFoundError:
-    long_description = DESCRIPTION
+    LONG_DESCRIPTION = DESCRIPTION
 
 # Load the package's version.py module as a dictionary.
-about = {}
+ABOUT = {}
 if not VERSION:
-    with open(os.path.join(here, NAME, 'version.py')) as f:
-        exec(f.read(), about)
+    with open(os.path.join(HERE, NAME, 'version.py')) as f:
+        exec(f.read(), ABOUT)   # pylint: disable=exec-used
 else:
-    about['__version__'] = VERSION
+    ABOUT['__version__'] = VERSION
 
 # Where the magic happens:
 setup(
     name=NAME,
-    version=about['__version__'],
+    version=ABOUT['__version__'],
     description=DESCRIPTION,
-    long_description=long_description,
+    long_description=LONG_DESCRIPTION,
     long_description_content_type='text/markdown',
     author=AUTHOR,
     author_email=EMAIL,
     python_requires=REQUIRES_PYTHON,
     url=URL,
     packages=find_packages(exclude=('tests', )),
-    setup_requires=SETUP_REQUIRED,
+    install_requires=REQUIRES,
     tests_require=TEST_REQUIRED,
     extras_require=EXTRAS,
     license='Apache License 2.0',

tests/__init__.py

@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+'''This module includes Fluid unit test cases'''