Adding zpm "ci" command to install from a lock file

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - #992: Implement automatic history purge logic
 - #973: Enables CORS and JWT configuration for WebApplications in module.xml
 - #1110: Add `iriscli` and `ipm` container utility scripts that are auto-installed to `~/.local/bin/` and `~/bin/` so they work both inside and outside of containers (Unix/Linux only)
+- #962: Adding zpm -from-lockfile flag to install/load from a lock file
 
 ### Fixed
 - #1001: The `unmap` and `enable` commands will now only activate CPF merge once after all namespaces have been configured instead after every namespace

src/cls/IPM/General/LockFile.cls

@@ -69,12 +69,12 @@ ClassMethod CreateLockFileForModule(
         $$$ThrowOnError(lockFile.Dependencies.SetAt(dependencyVal, mod.Name))
 
         // Add the dependency's repository to the lock file
-        do AddRepositoryToLockFile(.lockFile, mod.Repository, verbose)
+        do ..AddRepositoryToLockFile(.lockFile, mod.Repository, verbose)
     }
     // Add repository for base module if not already added by a dependency
     // Skip undefined repositories as that means the module was installed via the zpm "load" command
     if (module.Repository '= "") {
-        do AddRepositoryToLockFile(.lockFile, module.Repository, verbose)
+        do ..AddRepositoryToLockFile(.lockFile, module.Repository, verbose)
     }
 
     $$$ThrowOnError(lockFile.%JSONExportToStream(.lockFileJSON, "LockFileMapping"))
@@ -116,4 +116,93 @@ ClassMethod GetRepo(repoName As %String) As %IPM.Repo.Definition [ Internal ]
     }
 }
 
+ClassMethod InstallFromLockFile(
+	directory As %String,
+	ByRef params)
+{
+    set verbose = $get(params("Verbose"), 0)
+
+    set lockFilePath = ##class(%File).NormalizeFilename("module-lock.json", directory)
+    try {
+        set lockFileJSON = ##class(%DynamicObject).%FromJSONFile(lockFilePath)
+    } catch {
+        $$$ThrowStatus($$$ERROR($$$GeneralError,$$$FormatText("Unable to parse lock file JSON at path: %1", lockFilePath)))
+    }
+
+    set repositories = lockFileJSON.%Get("repositories", {})
+    set dependencies = lockFileJSON.%Get("dependencies", {})
+
+    // Install repositories (if they don't already exist)
+    set repoIter = repositories.%GetIterator()
+    while repoIter.%GetNext(.repoName, .repoVals) {
+        if ##class(%IPM.Repo.Definition).ServerDefinitionKeyExists(repoName) {
+            if (verbose) {
+                write !, "Repo: "_repoName_" already exists, skipping creating new one from lock file"
+            }
+            continue
+        } elseif (verbose) {
+            write !, "Creating repo: "_repoName_" from lock file"
+        }
+        do ##class(%IPM.Repo.Definition).CollectServerTypes(.types)
+        set repoClass = types(repoVals.type)
+        set repoVals.name = repoName
+        do $classmethod(repoClass, "LockFileValuesToModifiers", repoVals, .modifiers)
+        $$$ThrowOnError($classmethod(repoClass,"Configure",0,.modifiers,.tData,repoClass))
+    }
+
+    // Install modules (even if they already exist)
+    do ..GetOrderedDependenciesList(dependencies, .orderedDependenciesList)
+    set depName = ""
+    for i=1:1:$listlength(orderedDependenciesList) {
+        set depName = $list(orderedDependenciesList, i)
+        set depVals = dependencies.%Get(depName)
+
+        set version = depVals.version
+        set repository = depVals.repository
+
+        // Call Install() on dependency modules but set flag "LockFileInstallStarted" so we don't try installing from the dependency module's lock file
+        set commandInfo = "install"
+        set commandInfo("data", "Verbose") = verbose
+        set commandInfo("parameters","module") = repository_"/"_depName
+        set commandInfo("parameters", "version") = version
+        set commandInfo("data", "LockFileInstallStarted") = 1
+        set commandInfo("data","FromLockFile") = 1
+
+        do ##class(%IPM.Main).Install(.commandInfo)
+    }
+}
+
+/// The dependencies list in a lock file is listed alphabetically.
+/// This method compiles the dependencies and creates an ordered list such that no module is listed
+/// before one of its dependencies. Can then trace the list this outputs and install in order
+ClassMethod GetOrderedDependenciesList(
+	dependencies As %DynamicObject,
+	ByRef orderedDependenciesList As %List = "") [ Internal ]
+{
+    set depIter = dependencies.%GetIterator()
+    while depIter.%GetNext(.depName, .depVals) {
+        do ..AddToOrderedDependenciesList(depName, dependencies, .orderedDependenciesList)
+    }
+}
+
+/// For a dependency, recursively adds all dependencies to the ordered list, followed by this dependency
+ClassMethod AddToOrderedDependenciesList(
+	dependencyName As %String,
+	dependencies As %DynamicObject,
+	ByRef orderedDependenciesList As %List) [ Internal, Private ]
+{
+    set depVals = dependencies.%Get(dependencyName)
+    set transientDeps = depVals.%Get("dependencies", {})
+    set transientIter = transientDeps.%GetIterator()
+    while transientIter.%GetNext(.transDepName) {
+        // If dependency hasn't been installed yet, then recursively run this method on it
+        if '$listfind(orderedDependenciesList, transDepName) {
+            do ..AddToOrderedDependenciesList(transDepName, dependencies, .orderedDependenciesList)
+        }
+    }
+    if '$listfind(orderedDependenciesList, dependencyName) {
+        set orderedDependenciesList = orderedDependenciesList _ $listbuild(dependencyName)
+    }
+}
+
 }

src/cls/IPM/Lifecycle/Base.cls

@@ -1233,6 +1233,7 @@ Method %Export(
             "changelog.md",
             "license",
             "requirements.txt",
+            "module-lock.json",
         )
         set tRes = ##class(%File).FileSetFunc(..Module.Root)
         while tRes.%Next() {

src/cls/IPM/Main.cls

@@ -292,7 +292,8 @@ load C:\module\root\path -env C:\path\to\env1.json;C:\path\to\env2.json
 <modifier name="extra-pip-flags" dataAlias="ExtraPipFlags" value="true" description="Extra flags to pass to pip when installing python dependencies. Surround the flags (and values) with quotes if spaces are present. Default flags are &quot;--target &lt;target&gt; --python-version &lt;pyversion&gt; --only-binary=:all:&quot;." />
 <modifier name="synchronous" value="false" deprecated="true" description="DEPRECATED. Dependencies are now always loaded synchronously with independent lifecycle phases doing their own multi-threading as needed." />
 <modifier name="force" aliases="f" value="false" description="Allows the user to load a newer version of an existing module without running update steps." />
-<modifier name="create-lockfile" aliases="lock" dataAlias="CreateLockFile" dataValue="1" description="Upon load, creates/updates the module's lock file." />
+<modifier name="create-lockfile" dataAlias="CreateLockFile" dataValue="1" description="Upon load, creates/updates the module's lock file." />
+<modifier name="from-lockfile" dataAlias="FromLockFile" dataValue="1" description="Load the module from the lock file present at the path." />
 
 <!-- Parameters -->
 <parameter name="path" required="true" description="Directory on the local filesystem, containing a file named module.xml" />
@@ -419,7 +420,8 @@ install -env /path/to/env1.json;/path/to/env2.json example-package
 <modifier name="extra-pip-flags" dataAlias="ExtraPipFlags" value="true" description="Extra flags to pass to pip when installing python dependencies. Surround the flags (and values) with quotes if spaces are present. Default flags are &quot;--target &lt;target&gt; --python-version &lt;pyversion&gt; --only-binary=:all:&quot;."/>
 <modifier name="synchronous" value="false" deprecated="true" description="DEPRECATED. Dependencies are now always loaded synchronously with independent lifecycle phases doing their own multi-threading as needed." />
 <modifier name="force" aliases="f" value="false" description="Allows the user to install a newer version of an existing module without running update steps." />
-<modifier name="create-lockfile" aliases="lock" dataAlias="CreateLockFile" dataValue="1" description="Upon install, creates/updates the module's lock file." />
+<modifier name="create-lockfile" dataAlias="CreateLockFile" dataValue="1" description="Upon install, creates/updates the module's lock file." />
+<modifier name="from-lockfile" dataAlias="FromLockFile" dataValue="1" description="Load the module from the lock file present at the path." />
 
 </command>
 
@@ -1080,8 +1082,8 @@ ClassMethod ShellInternal(
                 do ..ModuleVersion(.tCommandInfo)
             } elseif (tCommandInfo = "information") {
                 do ..Information(.tCommandInfo)
-			} elseif (tCommandInfo = "history") {
-				do ..History(.tCommandInfo)
+            } elseif (tCommandInfo = "history") {
+                do ..History(.tCommandInfo)
             }
         } catch pException {
             if (pException.Code = $$$ERCTRLC) {
@@ -2377,14 +2379,18 @@ ClassMethod LoadInternal(
         }
     }
 
-    // When loading a module from a local folder, there might be a <mod root>/.modules/ folder containining dependencies.
-    // It's easier to configure a temporary repository than to handle this case in the dependency resolution code.
     set tTargetDirectory = $get(tTargetDirectory, tDirectoryName)
-    set dotModules = ##class(%File).NormalizeDirectory(".modules", tTargetDirectory)
-    set tmpRepoMgr = ##class(%IPM.General.TempLocalRepoManager).%New(dotModules, 1)
-    set tSC = ##class(%IPM.Utils.Module).LoadNewModule(tTargetDirectory, .tParams, , pLog)
-    set tSC = $$$ADDSC(tSC, tmpRepoMgr.CleanUp())
-    $$$ThrowOnError(tSC)
+    if ($get(pCommandInfo("data", "FromLockFile"))) {
+        do ##class(%IPM.General.LockFile).InstallFromLockFile(tTargetDirectory, .tParams)
+    } else {
+        // When loading a module from a local folder, there might be a <mod root>/.modules/ folder containining dependencies.
+        // It's easier to configure a temporary repository than to handle this case in the dependency resolution code.
+        set dotModules = ##class(%File).NormalizeDirectory(".modules", tTargetDirectory)
+        set tmpRepoMgr = ##class(%IPM.General.TempLocalRepoManager).%New(dotModules, 1)
+        set tSC = ##class(%IPM.Utils.Module).LoadNewModule(tTargetDirectory, .tParams, , pLog)
+        set tSC = $$$ADDSC(tSC, tmpRepoMgr.CleanUp())
+        $$$ThrowOnError(tSC)
+    }
 }
 
 ClassMethod CheckModuleNamespace() As %Status
@@ -2491,12 +2497,11 @@ ClassMethod Install(
                         $$$ThrowStatus($$$ERROR($$$GeneralError, "Deployed package '" _ tModuleName _ "' " _ tResult.VersionString _ " not supported on this platform " _ platformVersion _ "."))
                     }
                 }
-				$$$ThrowOnError(log.SetSource(tResult.ServerName))
-				$$$ThrowOnError(log.SetVersion(tResult.Version))
+                $$$ThrowOnError(log.SetSource(tResult.ServerName))
+                $$$ThrowOnError(log.SetVersion(tResult.Version))
                 $$$ThrowOnError(##class(%IPM.Utils.Module).LoadQualifiedReference(tResult, .tParams, , log))
             }
         } else {
-            set tPrefix = ""
             if (tModuleName '= "") {
                 if (tVersion '= "") {
                     $$$ThrowStatus($$$ERROR($$$GeneralError, tModuleName_" "_tVersion_" not found in any repository."))
@@ -2510,10 +2515,10 @@ ClassMethod Install(
             }
         }
     } catch ex {
-		$$$ThrowOnError(log.Finalize(ex.AsStatus(), devMode))
-		throw ex
-	}
-	$$$ThrowOnError(log.Finalize($$$OK, devMode))
+        $$$ThrowOnError(log.Finalize(ex.AsStatus(), devMode))
+        throw ex
+    }
+    $$$ThrowOnError(log.Finalize($$$OK, devMode))
 }
 
 ClassMethod Reinstall(ByRef pCommandInfo) [ Internal ]

src/cls/IPM/Repo/Definition.cls

@@ -288,6 +288,16 @@ Method LockFileTypeGet()
     return ..#MONIKER
 }
 
+/// We use the "LockFileMapping" XData block to export a repo definition to a lock file.
+/// When importing from a lock file, doing an import and save using the same XData block won't work.
+/// Instead, we must create a set of modifiers and call %IPM.Repo.Definition::Configure()
+/// This method accepts the JSON repository values from a lock file and populates a modifers object to then call Configure() with
+ClassMethod LockFileValuesToModifiers(
+	lockFileValues As %DynamicObject,
+	Output modifiers) [ Abstract ]
+{
+}
+
 Storage Default
 {
 <Data name="RepoDefinitionDefaultData">

src/cls/IPM/Repo/Filesystem/Definition.cls

@@ -357,11 +357,21 @@ ClassMethod ScanDirectory(
     quit tSC
 }
 
+ClassMethod LockFileValuesToModifiers(
+	lockFileValues As %DynamicObject,
+	Output modifiers)
+{
+    set modifiers("filesystem") = ""
+    set modifiers("name") = lockFileValues.%Get("name")
+    set modifiers("path") = lockFileValues.%Get("root")
+    set modifiers("depth") = lockFileValues.%Get("depth")
+    set modifiers("read-only") = lockFileValues.%Get("readOnly")
+}
+
 XData LockFileMapping
 {
 <Mapping xmlns="http://www.intersystems.com/jsonmapping">
   <Property Name="LockFileType" FieldName="type" />
-  <Property Name="OverriddenSortOrder" FieldName="overriddenSortOrder" />
   <Property Name="ReadOnly" FieldName="readOnly" />
   <Property Name="Root" FieldName="root" />
   <Property Name="Depth" FieldName="depth" />

src/cls/IPM/Repo/Oras/Definition.cls

@@ -140,11 +140,34 @@ Method GetPublishingManager(ByRef status)
     return ##class(%IPM.Repo.Oras.PublishManager).%Get(.status)
 }
 
+ClassMethod LockFileValuesToModifiers(
+	lockFileValues As %DynamicObject,
+	Output modifiers)
+{
+    set modifiers("oras") = ""
+    set modifiers("name") = lockFileValues.%Get("name")
+    set modifiers("read-only") = lockFileValues.%Get("readOnly")
+    set modifiers("url") = lockFileValues.%Get("url")
+    set modifiers("namespace") = lockFileValues.%Get("orasNamespace")
+
+    // The following variables are set as system level variables for us to get
+    // Naming convention for those follow the name of the repository, first converting any '-' to '_',
+    // then removing everything except for alphabetic characters, numbers, and '_' to use as variable prefix
+    // lastly, adds a suffix based on the modifier
+    // Examples: <repo name>        - <prefix>
+    //           "registry"         - "registry"
+    //           "ORAS!Repo(5)?"    - "ORASRepo5"
+    //           "My-Repository-2"  - "My_Repository_2"
+    set prefix = $zstrip($replace(modifiers("name"), "-", "_"), "*E'N'A")
+    set modifiers("username") = $system.Util.GetEnviron(prefix_"_username")
+    set modifiers("password") = $system.Util.GetEnviron(prefix_"_password")
+    set modifiers("token") = $system.Util.GetEnviron(prefix_"_token")
+}
+
 XData LockFileMapping
 {
 <Mapping xmlns="http://www.intersystems.com/jsonmapping">
   <Property Name="LockFileType" FieldName="type" />
-  <Property Name="OverriddenSortOrder" FieldName="overriddenSortOrder" />
   <Property Name="ReadOnly" FieldName="readOnly" />
   <Property Name="URL" FieldName="url" />
   <Property Name="Namespace" FieldName="orasNamespace" />

src/cls/IPM/Repo/Remote/Definition.cls

@@ -132,6 +132,29 @@ Method GetPublishingManager(ByRef status)
     return ##class(%IPM.Repo.Remote.PublishManager).%Get(.status)
 }
 
+ClassMethod LockFileValuesToModifiers(
+	lockFileValues As %DynamicObject,
+	Output modifiers)
+{
+    set modifiers("remote") = ""
+    set modifiers("name") = lockFileValues.%Get("name")
+    set modifiers("url") = lockFileValues.%Get("url")
+    set modifiers("read-only") = lockFileValues.%Get("readOnly")
+
+    // The following variables are set as system level variables for us to get
+    // Naming convention for those follow the name of the repository, first converting any '-' to '_',
+    // then removing everything except for alphabetic characters, numbers, and '_' to use as variable prefix
+    // lastly, adds a suffix based on the modifier
+    // Examples: <repo name>        - <prefix>
+    //           "registry"         - "registry"
+    //           "ORAS!Repo(5)?"    - "ORASRepo5"
+    //           "My-Repository-2"  - "My_Repository_2"
+    set prefix = $zstrip($replace(modifiers("name"), "-", "_"), "*E'N'A")
+    set modifiers("username") = $system.Util.GetEnviron(prefix_"_username")
+    set modifiers("password") = $system.Util.GetEnviron(prefix_"_password")
+    set modifiers("token") = $system.Util.GetEnviron(prefix_"_token")
+}
+
 Method LockFileTypeGet()
 {
     return ..#MONIKERALIAS
@@ -141,7 +164,6 @@ XData LockFileMapping
 {
 <Mapping xmlns="http://www.intersystems.com/jsonmapping">
   <Property Name="LockFileType" FieldName="type" />
-  <Property Name="OverriddenSortOrder" FieldName="overriddenSortOrder" />
   <Property Name="ReadOnly" FieldName="readOnly" />
   <Property Name="URL" FieldName="url" />
 </Mapping>

src/cls/IPM/Utils/Module.cls

@@ -271,6 +271,9 @@ ClassMethod LoadModuleFromDirectory(
 {
     set tSC = $$$OK
     try {
+        if $get(pParams("FromLockFile"), 0) && '$get(pParams("LockFileInstallStarted"), 0) {
+            do ##class(%IPM.General.LockFile).InstallFromLockFile(pDirectory, .pParams)
+        }
         set tVerbose = $get(pParams("Verbose"))
         // LoadNewModule goes all the way through Reload->Validate->Compile->Activate, also compiling the new module.
         write:tVerbose !,"Loading from ",pDirectory,!
@@ -991,7 +994,6 @@ ClassMethod GetModuleNameFromXML(
 ///     <Parameter Name="NoLock">1</Parameter>
 /// </Defaults>
 /// ```
-/// 
 /// Returns results as multidimensional array
 ClassMethod GetModuleDefaultsFromXML(
 	pDirectory As %String,
@@ -1248,7 +1250,12 @@ ClassMethod LoadNewModule(
         if $get(params("CreateLockFile"), 0) && '$data(params("LockFileModule")){
             set params("LockFileModule") = tModule.Name
         }
-        do ..LoadDependencies(tModule,, .params)
+
+        // If installing from a lock file, don't need to load dependencies since dependencies will be installed in order anyways
+        if ('$get(params("FromLockFile"), 0)) {
+            do ..LoadDependencies(tModule,,.params)
+        }
+
 
         set tSC = $system.OBJ.Load(pDirectory_"module.xml",$select(tVerbose:"d",1:"-d"),,.tLoadedList)
         $$$ThrowOnError(tSC)