Merge pull request #353 from zurk/feature/style-check

Add style check stage

Author: vmarkovtsev

.flake8

@@ -1,2 +1,19 @@
 [flake8]
-max-line-length=99
+ignore=B008,E121,E123,E126,E226,E24,E704,W503,W504,D100,D105,D200,D301,D402
+max-line-length=99
+exclude=
+    .git
+    doc
+inline-quotes="
+import-order-style=appnexus
+application-package-names=sourced.ml
+per-file-ignores=
+    **/tests/**:D
+    # Should be resolved one by one
+    # Related issue: https://github.com/src-d/ml/issues/354
+    ./sourced/ml/transformers/*:D
+    ./sourced/ml/extractors/*:D
+    ./sourced/ml/models/**:D
+    ./sourced/ml/algorithms/**:D
+    ./sourced/ml/utils/*:D
+    ./sourced/ml/cmd/*:D

.pylintrc

@@ -0,0 +1,10 @@
+[MASTER]
+jobs=0
+load-plugins=pylint.extensions.docparams
+
+[MESSAGES CONTROL]
+disable=all
+enable=missing-param-doc,
+       differing-param-doc,
+       differing-type-doc,
+       missing-return-doc

.travis.yml

@@ -7,6 +7,7 @@ cache:
   directories:
   - "$HOME/.cache/pip"
 stages:
+  - style
   - test
   - name: deploy
     # require any tag name to deploy
@@ -18,7 +19,8 @@ addons:
 _install: &_install
   - gimme 1.8
   - source ~/.gimme/envs/latest.env
-  - pip install --upgrade pip
+  # We need pip v18.1 to make python 3.4 work well
+  - (wget -O - https://bootstrap.pypa.io/get-pip.py || wget -O - https://raw.githubusercontent.com/pypa/get-pip/master/get-pip.py) | python3 - pip==18.1
   - if [[ $TRAVIS_PYTHON_VERSION < 3.7 ]]; then pip install tensorflow; fi
   - pip install -r requirements.txt codecov
   - pip install -e .[pandas]
@@ -32,7 +34,13 @@ _deploy: &_deploy
     tags: true
 matrix:
   include:
-    - python: 3.4
+    - stage: style
+      python: 3.7
+      env: SCRIPT="make check && cd doc && make"
+      install:
+      - pip install -r doc/requirements.txt -r requirements-lint.txt
+    - stage: test
+      python: 3.4
       dist: trusty
       env: *_coverage
       install: *_install
@@ -46,18 +54,13 @@ matrix:
       install: *_install
       after_success:
         - codecov
-    - python: 3.7
-      env: *_coverage
-      install: *_install
-      env: SCRIPT="flake8 . && (! grep -R /tmp sourced/ml/tests)"
-      install: pip install flake8
     - python: 3.7
       env: *_coverage
       install: *_install
     - stage: deploy
       python: 3.4
       install:
-        - pip3 install --upgrade pip
+        - (wget -O - https://bootstrap.pypa.io/get-pip.py || wget -O - https://raw.githubusercontent.com/pypa/get-pip/master/get-pip.py) | python3 - --user pip==18.1
         - pip3 install twine
       before_script: skip
       script:

Makefile

@@ -0,0 +1,13 @@
+.PHONY: check
+check:
+	! grep -R /tmp sourced/ml/tests
+	flake8 --count
+	pylint sourced.ml
+
+.PHONY: docker-build
+docker-build:
+	docker build -t srcd/ml .
+
+.PHONY: docker-build-core
+docker-build-core:
+	docker build -t srcd/ml-core --file Dockerfile.core .

doc/requirements.txt

@@ -0,0 +1,2 @@
+Sphinx==1.7.7
+sphinx-autoapi==0.6.0

requirements-lint.txt

@@ -0,0 +1,7 @@
+flake8==3.5.0
+flake8-bugbear==18.8.0
+flake8-docstrings==1.3.0
+flake8-import-order==0.18
+flake8-quotes==1.0.0
+flake8-per-file-ignores==0.6
+pylint==2.1.1

setup.py

@@ -1,12 +1,13 @@
+from importlib.machinery import SourceFileLoader
 import io
 import os.path
-from importlib.machinery import SourceFileLoader
-from setuptools import setup, find_packages
+
+from setuptools import find_packages, setup
 
 sourcedml = SourceFileLoader("sourced.ml", "./sourced/ml/__init__.py").load_module()
 
 with io.open(os.path.join(os.path.dirname(__file__), "README.md"),
-             encoding='utf-8') as f:
+             encoding="utf-8") as f:
     long_description = f.read()
 
 setup(

sourced/__init__.py

@@ -1,3 +1,4 @@
+"""Common namespace for sourced tools."""
 # You must not include any other code and data in a namespace package's __init__.py
 import pkg_resources
 pkg_resources.declare_namespace(__name__)

sourced/ml/__init__.py

@@ -1,3 +1,4 @@
+"""MLonCode research playground."""
 try:
     import modelforge.configuration
 

sourced/ml/__main__.py

@@ -5,20 +5,19 @@
 
 from modelforge import slogging
 
-from sourced.ml import extractors
-from sourced.ml.transformers import Moder
 from sourced.ml import cmd
+from sourced.ml import extractors
 from sourced.ml.cmd import args
 from sourced.ml.cmd.run_swivel import mirror_tf_args
+from sourced.ml.transformers import Moder
 from sourced.ml.utils import install_bigartm
 from sourced.ml.utils.spark import add_spark_args
 
 
 def get_parser() -> argparse.ArgumentParser:
     """
-    Creates the cmdline argument parser.
+    Create the cmdline argument parser.
     """
-
     parser = argparse.ArgumentParser(formatter_class=args.ArgumentDefaultsHelpFormatterNoNone)
     parser.add_argument("--log-level", default="INFO",
                         choices=logging._nameToLevel,
@@ -337,10 +336,10 @@ def add_parser(name, help_message):
 
 def main():
     """
-    Creates all the argparse-rs and invokes the function from set_defaults().
+    Create all the argparse-rs and invokes the function from set_defaults().
+
     :return: The result of the function from set_defaults().
     """
-
     parser = get_parser()
     args = parser.parse_args()
     args.log_level = logging._nameToLevel[args.log_level]