Merge pull request #287 from skhoroshavin/rc-1.0.1

 ST-609: RC 1.0.1

Author: skhoroshavin

.gitignore

@@ -23,6 +23,7 @@ lib64/
 parts/
 sdist/
 var/
+tails/
 *.egg-info/
 .installed.cfg
 *.egg

devops/Makefile

@@ -53,7 +53,7 @@ ifeq ($(SRC_DIR_NAME),sovtoken)
 # 	pypi: indy-plenum
 # 	apt: indy-plenum (stable component)
 
-FPM_P_DEPENDS := indy-node(=1.9.0)
+FPM_P_DEPENDS := indy-node(=1.9.1~rc1)
 FPM_ARGS := --no-python-dependencies $(FPM_ARGS)
 endif
 

devops/aws-codebuild/Jenkinsfile.cd

@@ -6,18 +6,34 @@ def sovLibrary = library(identifier: 'sovrin-aws-codebuild@v1.0.1', retriever: m
     github(credentialsId: 'sovbot-github', repoOwner: 'sovrin-foundation', repository: 'aws-codebuild-pipeline-plugin')
 )).com.sovrin.pipeline
 
+def getParam(name) {
+    return (params."$name" ?: env."$name")
+}
+
+def getApprove(name) {
+    def approval
+    try {
+        input(message: 'Do you want to publish this package?')
+        approval = true
+        echo "$name approval granted"
+    }
+    catch (Exception err) {
+        approval = false
+        echo "$name approval denied"
+    }
+    return approval
+}
+
 logger = sovLibrary.Logger.new(this)
 notifier = sovLibrary.Notifier.new(this)
+utils = sovLibrary.Utils.new(this)
 logger.setGlobalLevel('TRACE')
 
-gitHubUserCredId = env.GITHUB_BOT_USER ?: 'sovbot-github'
-sovrinPackagingRepo = env.SOVRIN_PACKAGING_REPO ?: 'https://github.com/sovrin-foundation/sovrin-packaging'
-sovrinPackagingBranch = env.SOVRIN_PACKAGING_BRANCH ?: 'master'
-
-def downloadPackagingUtils() {
-    git branch: sovrinPackagingBranch, credentialsId: gitHubUserCredId, url: sovrinPackagingRepo
-    sh "pip3 install -U plumbum deb-pkg-tools"
-}
+String gitHubUserCredId = getParam('GITHUB_BOT_USER') ?: 'sovbot-github'
+String sovrinPackagingRepo = getParam('env.SOVRIN_PACKAGING_REPO') ?: 'https://github.com/sovrin-foundation/sovrin-packaging'
+String sovrinPackagingBranch = getParam('SOVRIN_PACKAGING_BRANCH') ?: 'master'
+String repoName = getParam('SOVRIN_CORE_REPO_NAME') ?: 'core'
+String repoComponent = getParam('SOVRIN_CORE_REPO_CHANNEL') ?: env.BRANCH_NAME ?: 'master'
 
 def nodeLabels = [
     codeBuild: env.LIBSOVTOKEN_CODEBUILD_NODE_LABEL ?: 'codebuild',
@@ -30,35 +46,71 @@ pipelineWrapper({
     //this IP is outdated, should use the one from env
 //    env.SOVRIN_REPO_HOST = '192.168.11.115'
 
+    Boolean isRC = false
+    Boolean isRelease = false
+    Boolean isDev = (env.BRANCH_NAME == 'master')
+    List pkgsList = []
+    String debsDir = 'debs'
+    String pkgUtilsDir = 'sovrin-packaging'
+
+    Closure _upload = {
+        logger.info("Uploading debian packages to '${repoName}/${repoComponent}' sovrin repo: $pkgsList")
+        String u_id = utils.shStdout('id -u')
+        docker.build(
+                "$projectName:debs-upload",
+                "--build-arg=u_id=$u_id -f devops/docker/cd/debs_upload.dockerfile devops/docker/cd"
+        ).inside {
+            withCredentials([file(credentialsId: 'SovrinRepoSSHKey', variable: 'sovrin_key')]) {
+                sh """
+                    ./$pkgUtilsDir/upload_debs.py $debsDir $repoName $repoComponent --host $env.SOVRIN_REPO_HOST --ssh-key $sovrin_key
+                """
+            }
+        }
+    }
+
+    Closure _notify = {
+        notifier.email {
+            to = '$DEFAULT_RECIPIENTS ' + (this.env.LIBSOVTOKEN_DEF_RECIPIENTS ?: '')
+            subject = "[$projectName] new packages ${pkgsList.join(' ')} published to '${repoName}/${repoComponent}' repo"
+            body = """
+                |New debian packages ${pkgsList.join(' ')} were built and published to '${repoName}/${repoComponent}' repo
+                |
+                |Build:
+                |   Project: \$PROJECT_NAME
+                |   Url: ${this.env.BUILD_URL}
+                |   Number: ${this.env.BUILD_NUMBER}
+                |
+                |Check console output at ${this.env.BUILD_URL}console to view the details.
+            """.stripMargin()
+        }
+    }
+
     nodeWrapper(nodeLabels.codeBuild) {
         def osname = 'xenial'
         List projects = ['sovtoken', 'sovtokenfees']
-        List pkgsList = []
         String indyNodeVersion
-        Boolean isRC = false
-        Boolean isRelease = false
-        Boolean isDev = (env.BRANCH_NAME == 'master')
-
-        stage('Checkout sources from SCM') {
-            checkout scm
-        }
 
         def sovrinRepo = sovLibrary.SovrinRepo.new(this)
-        def utils = sovLibrary.Utils.new(this)
         def buildCtx = sovLibrary.AwsCodeBuildHelper.BuildCtx.new('plugin',
             ['devops', 'sovtoken/sovtoken/__metadata__.py', 'sovtokenfees/sovtokenfees/__metadata__.py'])
         def awsCBHelper = sovLibrary.AwsCodeBuildHelper.new(this, buildCtx)
 
-        stage('Get indy-node version') {
+        stage('Checkout sources from SCM') {
+            checkout scm
+        }
+
+        stage('Set release params') {
             indyNodeVersion = utils.shStdout("""
                 sed -n "s/.*indy-node==\\([^\\"\\']\\+\\).*/\\1/p" sovtoken/setup.py
             """)
             if (!indyNodeVersion) {
                 throw new Exception('indy-node version is not found')
             }
-            isRC = !!((env.BRANCH_NAME == 'stable') && indyNodeVersion.find(/[^0-9\.]/))
-            isRelease = ((env.BRANCH_NAME == 'stable') && !isRC)
-            logger.info("Detected indy-node version: '$indyNodeVersion', isRC: $isRC, isRelease: $isRelease")
+            logger.info("Detected indy-node version: '$indyNodeVersion'")
+
+            isRC = (env.BRANCH_NAME == 'stable')
+            isRelease = !!(isRC && !indyNodeVersion.find(/[^0-9\.]/))
+            logger.info("Release parameters: isRC: $isRC, isRelease: $isRelease")
         }
 
         Map builds = [
@@ -102,20 +154,21 @@ pipelineWrapper({
                         logger.info("Current source version: $srcVersion")
                     }
 
-//               stage('Get last revision') {
-//                   lastRevision = evernymRepo.getLastRevision {
-//                       delegate.packageName = packageName
-//                       packageSrcVersion = srcVersion
-//                       repoDistr = 'agency-dev'
-//                   }
-//
-//                   if (lastRevision) {
-//                       logger.info("Found last revision number: $lastRevision")
-//                   } else {
-//                       logger.info("No previous revision was found")
-//                   }
-//               }
+/*
+                    stage('Get last revision') {
+                        lastRevision = evernymRepo.getLastRevision {
+                            delegate.packageName = packageName
+                            packageSrcVersion = srcVersion
+                            repoDistr = 'agency-dev'
+                        }
 
+                        if (lastRevision) {
+                            logger.info("Found last revision number: $lastRevision")
+                        } else {
+                            logger.info("No previous revision was found")
+                        }
+                    }
+*/
 
                     stage('Set release parameters') {
                         def releaseVersion =  isRelease ? '' : (isRC ? 'rc' : isDev ? 'dev' : '') + "$BUILD_NUMBER"
@@ -158,14 +211,14 @@ pipelineWrapper({
 
                         onArtifacts = {
                             this.stage("$projLabel: Archive logs") {
-                                utils.archiveArtifacts("logs/*.log*") {
+                                this.utils.archiveArtifacts("logs/*.log*") {
                                     truncate = true
                                     allowEmptyArchive = true
                                 }
 
                                 String pkgBaseName = "${packageName}_${debPVersion}"
 
-                                this.stash includes: utils.shStdout("ls ${pkgBaseName}_*.deb"), name: pkgBaseName
+                                this.stash includes: this.utils.shStdout("ls ${pkgBaseName}_*.deb"), name: pkgBaseName
                                 pkgsList += pkgBaseName
                             }
                         }
@@ -177,48 +230,69 @@ pipelineWrapper({
         stage("Build artifacts") {
             builds.failFast = false
             utils.parallel builds
-        }
-
-        stage('Release artifacts') {
-            String repoName = env.SOVRIN_CORE_REPO_NAME
-            String repoComponent = env.BRANCH_NAME
-            String debsDir = 'debs'
-
-            if (isRC) {
-                repoComponent = 'rc'
-            }
-            logger.info("Debian repo component is set to: '$repoComponent'")
 
             dir(debsDir) {
                 pkgsList.each {
                     unstash name: it
                 }
             }
-            logger.info("Uploading debian packages to '${repoName}/${repoComponent}' sovrin repo: $pkgsList")
+            sh "ls -la $debsDir"
+        }
 
-            dir("sovrin-packaging") {
-                downloadPackagingUtils()
+        stage("Get packaging utils") {
+            dir(pkgUtilsDir) {
+                git branch: sovrinPackagingBranch, credentialsId: gitHubUserCredId, url: sovrinPackagingRepo
             }
+        }
 
-            withCredentials([file(credentialsId: 'SovrinRepoSSHKey', variable: 'sovrin_key')]) {
-                sh "./sovrin-packaging/upload_debs.py $debsDir $repoName $repoComponent --host $env.SOVRIN_REPO_HOST --ssh-key $sovrin_key --force-backup"
+        if (isRC) {
+            repoComponent = 'rc'
+        }
+
+        stage("Publish to $repoComponent") {
+            _upload()
+            if (isRelease) {
+                stash includes: "$pkgUtilsDir/**/*", name: pkgUtilsDir
+                stash includes: "$debsDir/**/*", name: debsDir
+            }
+        }
+    }
+
+    stage("Notify about $repoComponent") {
+        _notify()
+    }
+
+    if (isRelease) {
+        Boolean approval = false
+        repoComponent = 'stable'
+
+        stage("QA approval") {
+            approval = getApprove("QA")
+        }
+
+        if (!approval) {
+            return
+        }
+
+        nodeWrapper('ubuntu') {
+            stage('Checkout sources from SCM') {
+                checkout scm
             }
 
-            notifier.email {
-                to = '$DEFAULT_RECIPIENTS ' + (this.env.LIBSOVTOKEN_DEF_RECIPIENTS ?: '')
-                subject = "[$projectName] new packages ${pkgsList.join(' ')} published to '${repoName}/${repoComponent}' repo"
-                body = """
-                    |New debian packages ${pkgsList.join(' ')} were built and published to '${repoName}/${repoComponent}' repo
-                    |
-                    |Build:
-                    |   Project: \$PROJECT_NAME
-                    |   Url: ${this.env.BUILD_URL}
-                    |   Number: ${this.env.BUILD_NUMBER}
-                    |
-                    |Check console output at ${this.env.BUILD_URL}console to view the details.
-                """.stripMargin()
+            // Note. it is assumed here that packages are already in a backup
+            // directory and their upload would be actually skipped since '--force-backup'
+            // is not used
+            stage("Publish to $repoComponent") {
+                unstash pkgUtilsDir
+                unstash debsDir
+
+                _upload()
             }
         }
+
+        stage("Notify about $repoComponent") {
+            _notify()
+        }
     }
 }, { err ->
     if (err) {

devops/build-scripts/xenial/Pool_Party/Dockerfile

@@ -62,7 +62,7 @@ ARG sovtokenfees_pkg_name=sovtokenfees_0.9.6_amd64.deb
 COPY ${sovtoken_pkg_name} /root
 COPY ${sovtokenfees_pkg_name} /root
 
-RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 68DB5E88
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CE7709D068DB5E88
 RUN echo "deb https://repo.sovrin.org/deb xenial $indy_stream" >> /etc/apt/sources.list
 
 RUN apt-get update -y && apt-get install -y \

devops/denv/Dockerfile

@@ -9,7 +9,7 @@ RUN apt-get update -y && apt-get install -y \
         apt-transport-https \
         ca-certificates
 
-RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 68DB5E88
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CE7709D068DB5E88
 
 RUN echo "deb https://repo.sovrin.org/sdk/deb xenial stable" >> /etc/apt/sources.list
 
@@ -59,7 +59,7 @@ WORKDIR /home/indy
 # indy-node along with supervisor
 ENV INDY_NODE_VERSION ${INDY_NODE_VERSION:-1.7.1}
 ARG indy_plenum_ver=1.7.1
-RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 68DB5E88
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CE7709D068DB5E88
 RUN echo "deb https://repo.sovrin.org/deb xenial rc" >> /etc/apt/sources.list \
     && apt-get update && apt-get install -y \
 	    supervisor \

devops/docker/cd/debs_upload.dockerfile

@@ -0,0 +1,30 @@
+FROM ubuntu:xenial
+
+# debian packages
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        ssh \
+        python3.5 \
+        python3-pip \
+        python-setuptools \
+    && rm -rf /var/lib/apt/lists/*
+
+# pypi based packages
+# issues with pip>=10:
+# https://github.com/pypa/pip/issues/5240
+# https://github.com/pypa/pip/issues/5221
+RUN python3 -m pip install -U \
+        pip \
+        setuptools \
+        virtualenv \
+        pipenv \
+        plumbum \
+        deb-pkg-tools \
+    && pip3 list
+
+# user
+ARG u_id=1000
+ARG u_name=user
+
+RUN if [ "$u_id" != "0" ]; then \
+        useradd -ms /bin/bash -u $u_id $u_name; \
+    fi

devops/docker/ci/xenial/Dockerfile

@@ -10,7 +10,7 @@ RUN apt-get update && apt-get install -y \
     && rm -rf /var/lib/apt/lists/*
 
 # rocksdb
-RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 68DB5E88 \
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CE7709D068DB5E88 \
     && add-apt-repository "deb https://repo.sovrin.org/deb xenial rc master" \
     && apt-get update && apt-get install -y \
         libbz2-dev \
@@ -44,4 +44,4 @@ RUN if [ "$u_id" != "0" ]; then \
 USER $u_id
 
 # TODO CMD ENTRYPOINT ...
-ENV CI_ENV_VERSION=0.18.0
+ENV CI_ENV_VERSION=0.19.0

devops/vagrant/desktop/scripts/setup.sh

@@ -11,7 +11,7 @@ sudo apt-get update -y && apt-get install -y \
         apt-transport-https \
         ca-certificates
 
-sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 68DB5E88
+sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CE7709D068DB5E88
 
 echo "deb https://repo.sovrin.org/sdk/deb xenial stable" >> /etc/apt/sources.list
 

docs/README.md

@@ -0,0 +1,7 @@
+# Sovtokenfees
+
+The following documents provide necessary information regarding fees:
+
+* [Sovtokenfees Data Structures](./fees_data_structures.md)
+* [Fees related auth_rule](./fees_auth_rules.md)
+* [Setting fees process](https://github.com/sovrin-foundation/libsovtoken/blob/master/doc/fees.md)