From b0fdce8f5d7646625c4186bf855e1cf4aba30c56 Mon Sep 17 00:00:00 2001 From: henry Date: Tue, 25 Jan 2022 15:16:30 -0600 Subject: [PATCH] use python script for uploading docs instead of using deploy ssh plugin --- Jenkinsfile | 89 +++++++++----- ci/docker/python/shared/requirements-ci.txt | 1 + utils/upload_docs.py | 122 ++++++++++++++++++++ 3 files changed, 185 insertions(+), 27 deletions(-) create mode 100644 utils/upload_docs.py diff --git a/Jenkinsfile b/Jenkinsfile index 8a27243c..a98de53c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -214,7 +214,7 @@ pipeline { booleanParam(name: "DEPLOY_DEVPI_PRODUCTION", defaultValue: false, description: "Deploy to production devpi on https://devpi.library.illinois.edu/production/release. Master branch Only") booleanParam(name: 'DEPLOY_PYPI', defaultValue: false, description: 'Deploy to pypi') booleanParam(name: "DEPLOY_DOCS", defaultValue: false, description: "Update online documentation. Master branch Only") - string(name: 'DEPLOY_DOCS_URL_SUBFOLDER', defaultValue: "packager", description: 'The directory that the docs should be saved under') +// string(name: 'DEPLOY_DOCS_URL_SUBFOLDER', defaultValue: "packager", description: 'The directory that the docs should be saved under') } stages { stage('Build') { @@ -1128,41 +1128,76 @@ pipeline { } } } - stage("Deploy Online Documentation") { + stage('Deploy Online Documentation') { when{ equals expected: true, actual: params.DEPLOY_DOCS + beforeAgent true + beforeInput true + } + agent { + dockerfile { + filename 'ci/docker/python/linux/jenkins/Dockerfile' + label 'linux && docker' + } + } + options{ + timeout(time: 1, unit: 'DAYS') + } + input { + message 'Update project documentation?' } - agent any steps{ - unstash "DOCS_ARCHIVE" - dir("build/docs/html/"){ - input 'Update project documentation?' - sshPublisher( - publishers: [ - sshPublisherDesc( - configName: 'apache-ns - lib-dccuser-updater', - sshLabel: [label: 'Linux'], - transfers: [sshTransfer(excludes: '', - execCommand: '', - execTimeout: 120000, - flatten: false, - makeEmptyDirs: false, - noDefaultExcludes: false, - patternSeparator: '[, ]+', - remoteDirectory: params.DEPLOY_DOCS_URL_SUBFOLDER, - remoteDirectorySDF: false, - removePrefix: '', - sourceFiles: '**')], - usePromotionTimestamp: false, - useWorkspaceInPromotion: false, - verbose: true - ) + unstash 'DOCS_ARCHIVE' + withCredentials([usernamePassword(credentialsId: 'dccdocs-server', passwordVariable: 'docsPassword', usernameVariable: 'docsUsername')]) { + sh 'python utils/upload_docs.py --username=$docsUsername --password=$docsPassword --subroute=packager build/docs/html apache-ns.library.illinois.edu' + } + } + post{ + cleanup{ + cleanWs( + deleteDirs: true, + patterns: [ + [pattern: 'build/', type: 'INCLUDE'], + [pattern: 'dist/', type: 'INCLUDE'], ] ) } } } - +// stage("Deploy Online Documentation") { +// when{ +// equals expected: true, actual: params.DEPLOY_DOCS +// } +// agent any +// steps{ +// unstash "DOCS_ARCHIVE" +// dir("build/docs/html/"){ +// input 'Update project documentation?' +// sshPublisher( +// publishers: [ +// sshPublisherDesc( +// configName: 'apache-ns - lib-dccuser-updater', +// sshLabel: [label: 'Linux'], +// transfers: [sshTransfer(excludes: '', +// execCommand: '', +// execTimeout: 120000, +// flatten: false, +// makeEmptyDirs: false, +// noDefaultExcludes: false, +// patternSeparator: '[, ]+', +// remoteDirectory: params.DEPLOY_DOCS_URL_SUBFOLDER, +// remoteDirectorySDF: false, +// removePrefix: '', +// sourceFiles: '**')], +// usePromotionTimestamp: false, +// useWorkspaceInPromotion: false, +// verbose: true +// ) +// ] +// ) +// } +// } +// } } } } diff --git a/ci/docker/python/shared/requirements-ci.txt b/ci/docker/python/shared/requirements-ci.txt index 2cb634ef..74d3c2e1 100644 --- a/ci/docker/python/shared/requirements-ci.txt +++ b/ci/docker/python/shared/requirements-ci.txt @@ -5,6 +5,7 @@ devpi-client flake8 lxml mypy +paramiko pydocstyle pykdu-compress>=0.1.7b2 pylint diff --git a/utils/upload_docs.py b/utils/upload_docs.py new file mode 100644 index 00000000..8a4cc2f5 --- /dev/null +++ b/utils/upload_docs.py @@ -0,0 +1,122 @@ +import os +import paramiko +import argparse +import stat + + +def locate_documentation_files(root_path): + if not os.path.exists(os.path.join(root_path, "index.html")): + raise FileNotFoundError(f"No index.html found in {root_path}") + for root, dirs, files in os.walk(root_path): + for f in files: + yield os.path.relpath(root, root_path), f + + +def get_arg_parser(): + parser = argparse.ArgumentParser(description="Upload documentation") + + parser.add_argument( + "source_dir", + help="Root directory that contains the documentation rendered as html" + ) + + parser.add_argument( + "host", + help="Root directory that contains the documentation rendered as html" + ) + + parser.add_argument("--port", default=22) + parser.add_argument("--username", required=True) + parser.add_argument("--password", required=True) + parser.add_argument("--server_output", default='/var/www/html/dccdocs') + parser.add_argument("--subroute", help="Subdirectory to deploy files to") + + return parser + + +def remove_files_from_directory(client: paramiko.SFTPClient, root): + for i in client.listdir_attr(root): + server_path = os.path.join(root, i.filename) + if stat.S_ISDIR(i.st_mode): + remove_files_from_directory(client, server_path) + elif stat.S_ISREG(i.st_mode): + client.remove(server_path) + else: + raise RuntimeError( + f"Don't know what to do with {server_path}" + ) + print(f"Removed {server_path}") + client.rmdir(root) + print(f"Removed {root}") + + +def main(): + parser = get_arg_parser() + args = parser.parse_args() + + with paramiko.Transport((args.host, args.port)) as transport: + transport.connect(None, args.username, args.password) + + with paramiko.SFTPClient.from_transport(transport) as sftp: + temp_dir = f"{args.subroute}.new" + old_dir = f"{args.subroute}.old" + + output_directory = os.path.join(args.server_output, args.subroute) + output_directory_tmp = os.path.join(args.server_output, temp_dir) + + # location to rename the file while working + old_directory = os.path.join( + args.server_output, old_dir + ) + + # Remove any temp directory + if temp_dir in sftp.listdir(args.server_output): + print(f"Found {temp_dir}. Removing") + remove_files_from_directory(sftp, output_directory_tmp) + + if old_dir in sftp.listdir(args.server_output): + print(f"Found {old_dir}. Removing") + remove_files_from_directory(sftp, old_directory) + + # Copy data over to new tmp directory with .new suffix + try: + sftp.mkdir(output_directory_tmp) + print("Uploading new data") + sftp.chdir(output_directory_tmp) + + for subdir, file_name in locate_documentation_files( + args.source_dir + ): + output = os.path.join(subdir, file_name) + print(output) + try: + sftp.stat(subdir) + except FileNotFoundError: + sftp.mkdir(subdir) + + sftp.put( + os.path.join(args.source_dir, subdir, file_name), + output + ) + + except OSError: + remove_files_from_directory(sftp, output_directory_tmp) + raise + + # swap the existing version with the new version + if args.subroute in sftp.listdir(args.server_output): + print(f"Renaming {output_directory} to {old_directory}") + sftp.rename(output_directory, old_directory) + + print(f"Renaming {output_directory_tmp} to {output_directory}") + sftp.rename(output_directory_tmp, output_directory) + + print("Successfully deployed") + + # When new data is has the final filename, Remove old version + if old_dir in sftp.listdir(args.server_output): + print(f"cleaning up {old_dir}.") + remove_files_from_directory(sftp, old_directory) + +if __name__ == '__main__': + main()