|
| 1 | +/** |
| 2 | + * If there is an upgrade for a Stack apply it |
| 3 | + */ |
| 4 | + |
| 5 | +const { randomInt } = require('../../../../housekeeper/utils') |
| 6 | + |
| 7 | +module.exports = { |
| 8 | + name: 'stackUpgrade', |
| 9 | + // startup: false |
| 10 | + startup: 45000, |
| 11 | + schedule: `${randomInt(0, 29)} * * * *`, // random time in first half of hour |
| 12 | + run: async function (app) { |
| 13 | + if (app.config.features.enabled('autoStackUpdate')) { |
| 14 | + const date = new Date() |
| 15 | + const hour = date.getUTCHours() |
| 16 | + const day = date.getUTCDay() |
| 17 | + app.log.info(`Starting Stack Upgrade/Restart Check, hour: ${hour} day: ${day}`) |
| 18 | + const projectList = await app.db.models.ProjectSettings.getProjectsToUpgrade(hour, day) |
| 19 | + if (projectList) { |
| 20 | + for (const project of projectList) { |
| 21 | + if (project.value.restartOnly) { // this might need to be a separate flag to make the query work |
| 22 | + try { |
| 23 | + app.log.info(`Restarting project ${project.Project.id} as scheduled`) |
| 24 | + await app.db.controllers.Project.setInflightState(project.Project, 'restarting') |
| 25 | + project.Project.state = 'running' |
| 26 | + await project.Project.save() |
| 27 | + await app.containers.restartFlows(project.Project) |
| 28 | + await app.auditLog.Project.project.restarted(null, null, project.Project) |
| 29 | + await app.db.controllers.Project.clearInflightState(project.Project) |
| 30 | + } catch (err) { |
| 31 | + app.log.info(`Problem restarting project ${project.Project.id} - ${err.toString()}`) |
| 32 | + } |
| 33 | + } else { |
| 34 | + // we should probably rate limit this to not restart lots of projects at once |
| 35 | + if (project.Project.ProjectStack.replacedBy) { |
| 36 | + // need to add audit logging |
| 37 | + try { |
| 38 | + const newStack = await app.db.models.ProjectStack.byId(project.Project.ProjectStack.replacedBy) |
| 39 | + app.log.info(`Updating project ${project.Project.id} to stack: '${newStack.hashid}'`) |
| 40 | + |
| 41 | + const suspendOptions = { |
| 42 | + skipBilling: true |
| 43 | + } |
| 44 | + |
| 45 | + app.db.controllers.Project.setInflightState(project.Project, 'starting') |
| 46 | + const result = await suspendProject(project.Project, suspendOptions) |
| 47 | + |
| 48 | + await project.Project.setProjectStack(newStack) |
| 49 | + await project.Project.save() |
| 50 | + |
| 51 | + await app.auditLog.Project.project.stack.changed(null, null, project.Project, newStack) |
| 52 | + |
| 53 | + await unSuspendProject(project.Project, result.resumeProject, result.targetState) |
| 54 | + } catch (err) { |
| 55 | + app.log.info(`Problem updating project ${project.Project.id} - ${err.toString()}`) |
| 56 | + } |
| 57 | + } |
| 58 | + } |
| 59 | + } |
| 60 | + } |
| 61 | + app.log.info('Ending Stack Upgrade Check') |
| 62 | + } |
| 63 | + |
| 64 | + async function suspendProject (project, options) { |
| 65 | + let resumeProject = false |
| 66 | + const targetState = project.state |
| 67 | + if (project.state !== 'suspended') { |
| 68 | + resumeProject = true |
| 69 | + app.log.info(`Stopping project ${project.id}`) |
| 70 | + await app.containers.stop(project, options) |
| 71 | + await app.auditLog.Project.project.suspended(null, null, project) |
| 72 | + } |
| 73 | + return { resumeProject, targetState } |
| 74 | + } |
| 75 | + |
| 76 | + async function unSuspendProject (project, resumeProject, targetState) { |
| 77 | + if (resumeProject) { |
| 78 | + app.log.info(`Restarting project ${project.id}`) |
| 79 | + project.state = targetState |
| 80 | + await project.save() |
| 81 | + // Ensure the project has the full stack object |
| 82 | + await project.reload() |
| 83 | + const startResult = await app.containers.start(project) |
| 84 | + startResult.started.then(async () => { |
| 85 | + await app.auditLog.Project.project.started(null, null, project) |
| 86 | + app.db.controllers.Project.clearInflightState(project) |
| 87 | + return true |
| 88 | + }).catch(err => { |
| 89 | + app.log.info(`Failed to restart project ${project.id}`) |
| 90 | + throw err |
| 91 | + }) |
| 92 | + } else { |
| 93 | + app.db.controllers.Project.clearInflightState(project) |
| 94 | + } |
| 95 | + } |
| 96 | + } |
| 97 | +} |
0 commit comments