diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6174cfd..b83b50b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,7 @@ jobs: name: Java build runs-on: ubuntu-latest env: - luceeVersion: 6.2.0.321 + luceeVersion: 6.2.2.91 #luceeVersionQuery: 6.0.3/all/jar steps: - uses: actions/checkout@v4 @@ -59,6 +59,7 @@ jobs: testAdditional: ${{ github.workspace }}/tests testSevices: mysql LUCEE_ADMIN_ENABLED: false + DEBUG: true - name: Run Lucee Test Suite (testLabels="data-provider-integration") uses: lucee/script-runner@main #continue-on-error: true @@ -68,11 +69,18 @@ jobs: luceeVersion: ${{ env.luceeVersion }} #luceeVersionQuery: ${{ env.luceeVersionQuery }} extensionDir: ${{ github.workspace }}/ + luceeCFConfig: ${{ github.workspace }}/devops/.CFconfig-update.json5 env: testLabels: data-provider-integration testAdditional: ${{ github.workspace }}/tests testSevices: mysql LUCEE_ADMIN_ENABLED: false + DEBUG: true + - name: debug failure + if: ${{ failure() }} + run: | + pwd + ls -lR ${{ github.workspace }} build: name: Docker build & publish diff --git a/.gitignore b/.gitignore index eaa151e..671ad65 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ update/WEB-INF update/log-maven-download.log update/missing-bundles.txt /tests/artifacts +/apps/updateserver/services/legacy/artifacts +/.claude +/test-output diff --git a/README.md b/README.md index 804ae16..fc2dd35 100644 --- a/README.md +++ b/README.md @@ -7,3 +7,31 @@ This repo contains the application code for the following services used by Lucee * https://extension.lucee.org https://luceeserver.atlassian.net/issues/?jql=labels%20%3D%20%22updates%22 + +# Local Development + +By default, the data provider is configured to work in production. + +Create an `.env file` with the following settings to work locally. + +```env +ALLOW_RELOAD=true +S3_CORE_ROOT=/var/local_s3/lucee_downloads/ +S3_EXTENSIONS_ROOT=/var/local_s3/lucee_ext/ +S3_BUNDLES_ROOT=/var/local_s3/lucee_bundles/ +UPDATE_PROVIDER=http://127.0.0.1:8889/rest/update/provider/ +UPDATE_PROVIDER_INT=http://update:8888/rest/update/provider/ # internal docker networking +EXTENSION_PROVIDER=http://127.0.0.1:8889/rest/extension/provider/ +EXTENSION_PROVIDER_INT=http://update:8888/rest/extension/provider/ # internal docker networking +DOWNLOADS_URL=http://download:8888/ +``` + +Create folders under `local_s3` + +- in `lucee_downloads` place a few sample Lucee full jars +- in `lucee_ext` place some sample extension lex files + +The run `docker compose up` + +Running locally, the downloads page is at http://127.0.0.1:8888/ +and the update provider is http://127.0.0.1:8889/rest/update/provider/list?extended=true \ No newline at end of file diff --git a/apps/download/Application.cfc b/apps/download/Application.cfc index 44de468..1e16707 100644 --- a/apps/download/Application.cfc +++ b/apps/download/Application.cfc @@ -7,34 +7,46 @@ component { disallowDoctypeDecl: true }; - //this.s3.accessKeyId = server.system.environment.S3_DOWNLOAD_ACCESS_KEY_ID; - //this.s3.awsSecretKey = server.system.environment.S3_DOWNLOAD_SECRET_KEY; - request.s3Root="s3:///extension-downloads/"; request.s3URL="https://s3-eu-west-1.amazonaws.com/extension-downloads/"; function onApplicationStart() { - lock name="configure-sentry" type="exclusive" timeout=5{ - // workaround for LDEV-5371 - // if LUCEE_LOGGING_FORCE_APPENDER=console is set, sentry is bypassed - var deploy = expandPath( '{lucee-server}/../deploy/' ); - var sentry_json = deploy & "/.CFconfig-sentry.json"; - if ( FileExists( sentry_json ) ){ - configImport( sentry_json, "server", "admin" ); - systemOutput("sentry.json loaded via configImport - LDEV-5371", true); - fileDelete( sentry_json ); + application.extensionMeta = deserializeJson( fileRead( "extensionMeta.json" ) ); + application.sentryDsn = server.system.environment.SENTRY_DSN ?: ""; + + var sentryLogger = new services.SentryLogger( + config = { + dsn = application.sentryDsn, + environment = server.system.environment.SENTRY_ENVIRONMENT ?: "production", + release = server.system.environment.SENTRY_RELEASE ?: "", + serverName = server.system.environment.SENTRY_SERVER_NAME ?: cgi.server_name } - } - application.extensionMeta = deserializeJson(fileRead("extensionMeta.json")); + ); + + application.sentryLogger = sentryLogger; } - function onError(e){ - if (cgi.script_name contains "admin" or cgi.script_name contains "lucee"){ + function onError( e ){ + if ( cgi.script_name contains "admin" or cgi.script_name contains "lucee" ){ header statuscode="418" statustext="nice try, you're a teapot"; return; } + + // Log to Sentry + try { + if ( structKeyExists( application, "sentryLogger" ) ) { + application.sentryLogger.logException( + exception = arguments.e, + level = "error" + ); + } + } catch ( any err ) { + // Don't let Sentry failures break error handling + systemOutput( "Failed to log error to Sentry: #err.message#", true ); + } + header statuscode="500" statustext="Server error"; - echo("Sorry, Server error"); + echo( "Sorry, Server error" ); } function onRequest( string requestedPath ) output=true { diff --git a/apps/download/download.cfc b/apps/download/download.cfc index 293f113..41550cc 100644 --- a/apps/download/download.cfc +++ b/apps/download/download.cfc @@ -1,29 +1,28 @@ component { - variables.EXTENSION_PROVIDER="https://extension.lucee.org/rest/extension/provider/info?withLogo=true&type=all"; - //variables.EXTENSION_PROVIDER="http://127.0.0.1:8889/rest/extension/provider/info?withLogo=true&type=all"; - - variables.EXTENSION_DOWNLOAD="https://extension.lucee.org/rest/extension/provider/{type}/{id}"; - - variables.UPDATE_PROVIDER = "https://update.lucee.org/rest/update/provider"; - - //variables.UPDATE_PROVIDER = "http://127.0.0.1:8889/rest/update/provider"; + // provider urls are to communicate between the docker instances, so they need different urls + // set EXTENSION_PROVIDER_INT=http://update:8888/rest/extension/provider in .env for local testing + variables.EXTENSION_PROVIDER = server.system.environment.EXTENSION_PROVIDER_INT ?: "https://extension.lucee.org/rest/extension/provider"; + + // set DOWNLOAD_UPDATE_PROVIDER_INT=http://update:8888/rest/update/provider in .env for local testing + variables.UPDATE_PROVIDER = server.system.environment.UPDATE_PROVIDER_INT ?: "https://update.lucee.org/rest/update/provider"; function getExtensions(flush=false) localmode=true { + var extUrl = EXTENSION_PROVIDER &"/info?withLogo=true&type=all"; if ( arguments.flush || isNull( application.extInfo ) ) { - http url=EXTENSION_PROVIDER&"&flush="&arguments.flush result="http"; + http url=extUrl result="http"; if ( isNull( http.status_code ) || http.status_code != 200 ) - throw "could not connect to extension provider (#EXTENSION_PROVIDER#)"; + throw "could not connect to extension provider (#extUrl#)"; var data = deSerializeJson( http.fileContent, false ); - if (!structKeyExists( data, "meta" ) ) { + if ( !structKeyExists( data, "meta" ) ) { systemOutput( "error fetching extensions, falling back on cache", true); - http url=EXTENSION_PROVIDER result="http"; + http url=extUrl result="http"; if ( isNull( http.status_code ) || http.status_code != 200 ) - throw "could not connect to extension provider (#EXTENSION_PROVIDER#)"; + throw "could not connect to extension provider (#extUrl#)"; data = deSerializeJson( http.fileContent, false ); application.extInfo = data.extensions; @@ -240,8 +239,12 @@ component { application[ "changelogLastUpdated" ] = lastUpdated; // application.mavenInfo = {}; // not currently used // maven dates are static, only purge if unknown - loop collection="#application.mavenDates#" key="local.version" value="local.date" { - if ( len(date) eq 0 ) structDelete( application.mavenDates, version ); + if ( structKeyExists( application, "mavenDates" ) ){ + loop collection="#application.mavenDates#" key="local.version" value="local.date" { + if ( len(date) eq 0 ) structDelete( application.mavenDates, version ); + } + } else { + application.mavenDates = {}; } } diff --git a/apps/download/index.cfm b/apps/download/index.cfm index d9f87a9..301df7f 100644 --- a/apps/download/index.cfm +++ b/apps/download/index.cfm @@ -42,7 +42,7 @@ lang.installer['linux-x64']="Linux (x64)"; lang.installer['linux-aarch64']="Linux (aarch64)"; - cdnURL="https://cdn.lucee.org/"; + cdnURL="https://cdn.lucee.org/"; // now comes from the provider cdnURLExt="https://ext.lucee.org/"; MAX=1000; @@ -165,11 +165,11 @@
- + -
  • Express +
  • Express @@ -180,8 +180,8 @@ - - #lang.installer[kk]# Installer
    • '> @@ -190,35 +190,35 @@ - + -
  • lucee.jar +
  • lucee.jar
    • - + -
  • lucee-light.jar +
  • lucee-light.jar
    • - + -
  • lucee-zero.jar +
  • lucee-zero.jar
    • @@ -226,12 +226,12 @@ - + -
  • Core +
  • Core
    • @@ -239,13 +239,13 @@ - +
  • - WAR + WAR @@ -367,12 +367,12 @@
    - +
    - Express + Express @@ -386,11 +386,12 @@ - + - #lang.installer[kk]# Installer ' + &' '> @@ -409,12 +410,12 @@
    - + -
    lucee.jar +
    lucee.jar @@ -422,12 +423,12 @@ - + -
    lucee-light.jar +
    lucee-light.jar @@ -435,12 +436,12 @@ - + -
    lucee-zero.jar +
    lucee-zero.jar @@ -451,11 +452,11 @@
    - + -
    Core +
    Core @@ -466,12 +467,12 @@
    - + -
    WAR +
    WAR @@ -630,7 +631,7 @@
    class="row_alterEven textStyle textWrap"class="row_alterOdd textStyle textWrap"> - title="Requires Lucee #encodeForHTMLAttribute(el.meta.mincoreversion)#" diff --git a/apps/download/services/SentryLogger.cfc b/apps/download/services/SentryLogger.cfc new file mode 100644 index 0000000..b041f12 --- /dev/null +++ b/apps/download/services/SentryLogger.cfc @@ -0,0 +1,375 @@ +/** + * A CFML component for logging events and errors to Sentry. + * Supports Sentry's Store API for sending events. + */ +component { + + /** + * Constructor for the SentryLogger component. + * + * @param config A struct containing the Sentry connection details. + * - dsn: Your Sentry DSN (e.g., "https://key@sentry.io/project-id"). + * - environment: The environment name (e.g., "production", "staging", "development"). Defaults to "production". + * - release: Optional release identifier. + * - serverName: Optional server name. Defaults to cgi.server_name. + */ + public function init( required struct config ) { + variables.config = arguments.config; + + // DSN is optional - if not provided or empty, logger will exercise all code paths but not send to Sentry + // Strip quotes in case DSN comes through as literal '""' string from environment variables + var cleanDsn = trim( replace( variables.config.dsn ?: "", '""', '', 'all' ) ); + if ( len( cleanDsn ) && cleanDsn neq '""' ) { + // Parse the DSN + variables.sentryInfo = _parseDsn( cleanDsn ); + variables.hasDsn = true; + } else { + variables.hasDsn = false; + } + + // Set defaults + if ( !structKeyExists( variables.config, "environment" ) || isEmpty( variables.config.environment ) ) { + variables.config.environment = "production"; + } + + if ( !structKeyExists( variables.config, "serverName" ) || isEmpty( variables.config.serverName ) ) { + variables.config.serverName = cgi.server_name ?: "unknown"; + } + + return this; + } + + /** + * Logs a message to Sentry. + * + * @param message The message to log. + * @param level The severity level: "debug", "info", "warning", "error", "fatal". Defaults to "info". + * @param extra Additional context data to send with the event. + * @param tags Tags to categorize the event. + * @param user User information (id, email, username, ip_address). + */ + public struct function logMessage( + required string message, + string level = "info", + struct extra = {}, + struct tags = {}, + struct user = {} + ) { + var event = _buildEvent( + message = arguments.message, + level = arguments.level, + extra = arguments.extra, + tags = arguments.tags, + user = arguments.user + ); + + return _sendEvent( event ); + } + + /** + * Logs an exception to Sentry. + * + * @param exception The exception object (cfcatch). + * @param level The severity level. Defaults to "error". + * @param extra Additional context data to send with the event. + * @param tags Tags to categorize the event. + * @param user User information. + */ + public struct function logException( + required any exception, + string level = "error", + struct extra = {}, + struct tags = {}, + struct user = {} + ) { + // Handle simple values (string, number, etc) passed as exception + var exceptionMessage = "Unknown error"; + var exceptionType = "Error"; + var exceptionStruct = arguments.exception; + + if ( !isStruct( arguments.exception ) ) { + // Simple value passed - convert to string + exceptionMessage = toString( arguments.exception ); + exceptionStruct = {}; + } else { + exceptionMessage = arguments.exception.message ?: toString( arguments.exception ); + exceptionType = arguments.exception.type ?: "Error"; + } + + // Use custom logText as the main message if provided, otherwise use exception message + var displayMessage = exceptionMessage; + if ( structKeyExists( arguments.extra, "logText" ) && len( arguments.extra.logText ) ) { + displayMessage = arguments.extra.logText; + // Keep the original exception message in extra context + arguments.extra[ "exceptionMessage" ] = exceptionMessage; + } + + var event = _buildEvent( + message = displayMessage, + level = arguments.level, + extra = arguments.extra, + tags = arguments.tags, + user = arguments.user + ); + + // Add exception details + event[ "exception" ] = { + "values" = [ + { + "type" = exceptionType, + "value" = exceptionMessage, + "stacktrace" = _parseStackTrace( exceptionStruct ) + } + ] + }; + + // Add additional exception context if it's a proper exception struct + if ( isStruct( arguments.exception ) ) { + if ( structKeyExists( arguments.exception, "detail" ) && len( arguments.exception.detail ) ) { + event.extra[ "detail" ] = arguments.exception.detail; + } + + if ( structKeyExists( arguments.exception, "extendedInfo" ) && len( arguments.exception.extendedInfo ) ) { + event.extra[ "extendedInfo" ] = arguments.exception.extendedInfo; + } + } + + return _sendEvent( event ); + } + + /** + * Builds a base event structure for Sentry. + */ + private struct function _buildEvent( + required string message, + required string level, + required struct extra, + required struct tags, + required struct user + ) { + var event = { + "event_id" = lCase( replace( createUUID(), "-", "", "all" ) ), + "timestamp" = _getIsoTimestamp(), + "level" = _normalizeSentryLevel( arguments.level ), + "message" = arguments.message, + "platform" = "cfml", + "environment" = variables.config.environment, + "server_name" = variables.config.serverName, + "extra" = arguments.extra, + "tags" = arguments.tags + }; + + // Add release if configured + if ( structKeyExists( variables.config, "release" ) && len( variables.config.release ) ) { + event[ "release" ] = variables.config.release; + } + + // Add user context if provided + if ( structCount( arguments.user ) ) { + event[ "user" ] = arguments.user; + } + + // Add request context + event[ "request" ] = _getRequestContext(); + + // Add server context + event[ "contexts" ] = _getServerContext(); + + return event; + } + + /** + * Sends an event to Sentry via HTTP. + */ + private struct function _sendEvent( required struct event ) { + // If no DSN configured, skip the HTTP call but return success + // This allows full code path testing in dev without sending to Sentry + if ( !variables.hasDsn ) { + return { + "success" = true, + "eventId" = arguments.event.event_id, + "statusCode" = "200", + "note" = "No DSN configured - event not sent to Sentry" + }; + } + + var fullUrl = variables.sentryInfo.apiUrl; + var timestamp = _getTimestamp(); + + try { + cfhttp( method="POST", url=fullUrl, result="local.result", throwOnError=false, timeout=5 ) { + cfhttpparam( type="header", name="Content-Type", value="application/json" ); + cfhttpparam( type="header", name="X-Sentry-Auth", value=_buildAuthHeader( timestamp ) ); + cfhttpparam( type="body", value=serializeJson( arguments.event ) ); + } + + if ( !find( "200", local.result.statusCode ) ) { + throw( + type = "Sentry.HTTPException", + message = "Sentry API returned status: #local.result.statusCode#", + detail = "Response: " & local.result.fileContent + ); + } + + return { + "success" = true, + "eventId" = arguments.event.event_id, + "statusCode" = local.result.statusCode + }; + + } catch ( any e ) { + // Don't let logging failures break the application + // You could log this to a file or dump it + return { + "success" = false, + "error" = e.message, + "detail" = e.detail ?: "" + }; + } + } + + /** + * Parses a Sentry DSN into its components. + */ + private struct function _parseDsn( required string dsn ) { + // DSN format: https://{publicKey}@{host}/{projectId} + var pattern = "^(https?)://([^@]+)@([^/]+)/(.+)$"; + var matches = reFind( pattern, arguments.dsn, 1, true ); + + if ( !matches.pos[ 1 ] ) { + throw( type="Sentry.ConfigurationException", message="Invalid DSN format" ); + } + + var protocol = mid( arguments.dsn, matches.pos[ 2 ], matches.len[ 2 ] ); + var publicKey = mid( arguments.dsn, matches.pos[ 3 ], matches.len[ 3 ] ); + var host = mid( arguments.dsn, matches.pos[ 4 ], matches.len[ 4 ] ); + var projectId = mid( arguments.dsn, matches.pos[ 5 ], matches.len[ 5 ] ); + + return { + "protocol" = protocol, + "publicKey" = publicKey, + "host" = host, + "projectId" = projectId, + "apiUrl" = "#protocol#://#host#/api/#projectId#/store/" + }; + } + + /** + * Builds the Sentry auth header. + */ + private string function _buildAuthHeader( required numeric timestamp ) { + var parts = [ + "Sentry sentry_version=7", + "sentry_client=cfml-sentry/1.0", + "sentry_timestamp=#arguments.timestamp#", + "sentry_key=#variables.sentryInfo.publicKey#" + ]; + + return arrayToList( parts, ", " ); + } + + /** + * Gets the current timestamp in seconds. + */ + private numeric function _getTimestamp() { + return fix( getTickCount() / 1000 ); + } + + /** + * Gets the current timestamp in ISO 8601 format. + */ + private string function _getIsoTimestamp() { + return dateTimeFormat( now(), "iso8601" ); + } + + /** + * Extracts request context from CGI scope. + */ + private struct function _getRequestContext() { + var context = { + "url" = cgi.server_name & cgi.script_name, + "method" = cgi.request_method, + "query_string" = cgi.query_string, + "headers" = {} + }; + + // Add common headers including user agent + var headerKeys = [ "user-agent", "referer", "content-type" ]; + for ( var key in headerKeys ) { + var cgiKey = "http_" & replace( key, "-", "_", "all" ); + if ( structKeyExists( cgi, cgiKey ) ) { + context.headers[ key ] = cgi[ cgiKey ]; + } + } + + return context; + } + + /** + * Gets server context including Lucee and Java versions. + */ + private struct function _getServerContext() { + var contexts = { + "runtime" = { + "name" = "Lucee", + "version" = server.lucee.version ?: "unknown" + }, + "os" = { + "name" = server.os.name ?: "unknown", + "version" = server.os.version ?: "unknown" + } + }; + + // Add Java version + if ( structKeyExists( server, "java" ) && structKeyExists( server.java, "version" ) ) { + contexts[ "runtime" ][ "java_version" ] = server.java.version; + } + + return contexts; + } + + /** + * Parses CFML exception stack trace into Sentry format. + */ + private struct function _parseStackTrace( required any exception ) { + var frames = []; + + if ( structKeyExists( arguments.exception, "tagContext" ) && isArray( arguments.exception.tagContext ) ) { + for ( var frame in arguments.exception.tagContext ) { + arrayAppend( frames, { + "filename" = frame.template ?: "", + "lineno" = frame.line ?: 0, + "function" = frame.id ?: "", + "in_app" = true + } ); + } + } + + return { + "frames" = frames + }; + } + + /** + * Normalizes level names to match Sentry's expected format. + * Handles common variations like "warn" -> "warning", "ERROR" -> "error", etc. + */ + private string function _normalizeSentryLevel( required string level ) { + var normalized = lCase( trim( arguments.level ) ); + + // Map common variations to Sentry levels + switch ( normalized ) { + case "warn": + return "warning"; + case "fatal": + case "critical": + return "fatal"; + case "err": + return "error"; + default: + // Return as-is for: debug, info, warning, error, fatal + return normalized; + } + } + +} diff --git a/apps/updateserver/Application.cfc b/apps/updateserver/Application.cfc index d244934..9c1ef6e 100644 --- a/apps/updateserver/Application.cfc +++ b/apps/updateserver/Application.cfc @@ -12,38 +12,36 @@ component { this.searchImplicitScopes = false; this.searchResults = false; this.scopeCascading = 'strict'; - this.s3.accessKeyId = server.system.environment.S3_EXTENSION_ACCESS_KEY_ID; - this.s3.awsSecretKey = server.system.environment.S3_EXTENSION_SECRET_KEY; + this.s3.accessKeyId = server.system.environment.S3_EXTENSION_ACCESS_KEY_ID ?: ""; + this.s3.awsSecretKey = server.system.environment.S3_EXTENSION_SECRET_KEY ?: ""; this.allowReload = IsBoolean( server.system.environment.ALLOW_RELOAD ?: "" ) && server.system.environment.ALLOW_RELOAD; function onApplicationStart() { - lock name="configure-sentry" type="exclusive" timeout=5{ - // workaround for LDEV-5371 - // if LUCEE_LOGGING_FORCE_APPENDER=console is set, sentry is bypassed - var deploy = expandPath( '{lucee-server}/../deploy/' ); - var sentry_json = deploy & "/.CFconfig-sentry.json"; - if ( FileExists( sentry_json ) ){ - configImport( sentry_json, "server", "admin" ); - systemOutput("sentry.json loaded via configImport - LDEV-5371", true); - fileDelete( sentry_json ); - } - } - _loadServices(); + systemOutput( "[#dateTimeFormat(now(), "long")#] onApplicationStart() called for lucee-provider", true ); + application.state = "init"; + + application.state = "starting"; + var success = _loadServices(); + application.state = success ? "loaded" : "failed"; + + systemOutput( "[#dateTimeFormat(now(), "long")#] onApplicationStart() finished, state: #application.state#", true ); + return success; } function onRequestStart() output=true { - var sentry_dsn = "https://o1289959.ingest.us.sentry.io/api/4507452800040960/security/?sentry_key=9af63ea401ee6935d34159019b1ae765"; var csp = [ "sandbox" , "object-src 'none'" , "script-src 'none'" - , "report-uri #sentry_dsn#" + , "report-uri #application.sentryDsn#" ]; header name="Content-Security-Policy" value=ArrayToList( csp, "; " ); if ( this.allowReload && StructKeyExists( url, "fwreinit" ) ) { - _loadServices(); + application.state = "starting"; + var success = _loadServices(); + application.state = success ? "loaded" : "failed"; } var allowedPaths = [ "rest", "healthcheck", "index.cfm" ]; @@ -57,28 +55,71 @@ component { } } + function onError( required any exception, required string eventName ) { + // Log uncaught errors to Sentry + try { + if ( structKeyExists( application, "sentryLogger" ) ) { + application.sentryLogger.logException( + exception = arguments.exception, + level = "error", + tags = { "eventName" = arguments.eventName } + ); + } + } catch ( any e ) { + // Don't let Sentry failures break error handling + systemOutput( "Failed to log error to Sentry: #e.message#", true ); + } + + // Rethrow the original exception so it's still logged locally + throw( + type = arguments.exception.type, + message = arguments.exception.message, + detail = arguments.exception.detail ?: "", + cause = arguments.exception + ); + } + function _loadServices() { setting requesttimeout=600; - var coreS3Root = server.system.environment.S3_CORE_ROOT ?: "s3:///lucee-downloads/"; - var coreCdnUrl = server.system.environment.S3_CORE_CDN_URL ?: "https://cdn.lucee.org/"; - var extS3Root = server.system.environment.S3_EXTENSIONS_ROOT ?: "s3:///extension-downloads/"; - var extCdnUrl = server.system.environment.S3_EXTENSIONS_CDN_URL ?: "https://ext.lucee.org/"; - var bundleS3Root = server.system.environment.S3_BUNDLES_ROOT ?: "s3:///bundle-download/"; - var bundleCdnUrl = server.system.environment.S3_BUNDLES_CDN_URL ?: "https://bundle.lucee.org/"; + application.coreS3Root = server.system.environment.S3_CORE_ROOT ?: "s3:///lucee-downloads/"; + application.coreCdnUrl = server.system.environment.S3_CORE_CDN_URL ?: "https://cdn.lucee.org/"; + application.extensionsS3Root = server.system.environment.S3_EXTENSIONS_ROOT ?: "s3:///extension-downloads/"; + application.extensionsCdnUrl = server.system.environment.S3_EXTENSIONS_CDN_URL ?: "https://ext.lucee.org/"; + application.bundleS3Root = server.system.environment.S3_BUNDLES_ROOT ?: "s3:///bundle-download/"; + application.bundleCdnUrl = server.system.environment.S3_BUNDLES_CDN_URL ?: "https://bundle.lucee.org/"; + application.downloadsUrl = server.system.environment.DOWNLOADS_URL ?: "https://download.lucee.org/"; + application.updateProviderUrl = server.system.environment.UPDATE_PROVIDER ?: "https://update.lucee.org/rest/update/provider/"; + application.extensionProviderUrl = server.system.environment.EXTENSION_PROVIDER ?: "https://extensions.lucee.org/rest/extension/provider/"; + application.sentryDsn = server.system.environment.SENTRY_DSN ?: ""; + + if ( left( application.coreS3Root, 3 ) == "s3:" + && ( len( this.s3.awsSecretKey ) == 0 || len( this.s3.accessKeyId ) == 0) ) { + var s3Error = "ERROR: S3 Credentials Required [ S3_EXTENSION_ACCESS_KEY_ID, S3_EXTENSION_SECRET_KEY ]," + & " for local testing set S3_CORE_ROOT [#application.coreS3Root#] to a directory"; + systemOutput( s3Error, true ); + return false; + } else { + // always delete local cache to start with a clean slate + var versionsCache ="services/legacy/cache/versions.json"; + if ( fileExists( versionsCache ) ){ + systemOutput("LOCAL_S3: purging version cache [#versionsCache#]", true); + fileDelete( versionsCache ); + } + } var extMetaReader = new services.ExtensionMetadataReader( - s3root = extS3Root + s3root = application.extensionsS3Root ); var jiraChangelogService = new services.JiraChangelogService( - s3root = coreS3Root + s3root = application.coreS3Root ); var bundleDownloadService = new services.BundleDownloadService( - extensionsCdnUrl = extCdnUrl - , bundleS3Root = bundleS3Root - , bundleCdnUrl = bundleCdnUrl + extensionsCdnUrl = application.extensionsCdnUrl + , bundleS3Root = application.bundleS3Root + , bundleCdnUrl = application.bundleCdnUrl , extensionMetaReader = extMetaReader , mavenMatcher = new services.legacy.MavenMatcher() ); @@ -90,13 +131,21 @@ component { jiraChangelogService.updateIssuesAsync(); new services.legacy.MavenRepo().list(); - application.coreS3Root = coreS3Root; - application.coreCdnUrl = coreCdnUrl; - application.extensionsCdnUrl = extCdnUrl; - application.extensionsS3Root = extS3Root; + var sentryLogger = new services.SentryLogger( + config = { + dsn = application.sentryDsn, + environment = server.system.environment.SENTRY_ENVIRONMENT ?: "production", + release = server.system.environment.SENTRY_RELEASE ?: "", + serverName = server.system.environment.SENTRY_SERVER_NAME ?: cgi.server_name + } + ); + application.extMetaReader = extMetaReader; application.bundleDownloadService = bundleDownloadService; application.jiraChangelogService = jiraChangelogService; + application.sentryLogger = sentryLogger; + + return true; } } \ No newline at end of file diff --git a/apps/updateserver/healthcheck/index.cfm b/apps/updateserver/healthcheck/index.cfm index 93caf3f..427bfb2 100644 --- a/apps/updateserver/healthcheck/index.cfm +++ b/apps/updateserver/healthcheck/index.cfm @@ -1 +1,11 @@ -#echo("OK")# \ No newline at end of file + + if ( !structKeyExists( application, "state" ) || application.state != "loaded" ) { + var state = application.state ?: 'undefined'; + systemOutput( "healthcheck FAILED: Application not ready. State: #state#", true ); + header statuscode="503" statustext="Service Unavailable"; + echo( "Application not ready. State: #state#" ); + abort; + } + + echo( "OK" ); + \ No newline at end of file diff --git a/apps/updateserver/rest/Application.cfc b/apps/updateserver/rest/Application.cfc index aef10e4..e4fa112 100644 --- a/apps/updateserver/rest/Application.cfc +++ b/apps/updateserver/rest/Application.cfc @@ -1,3 +1,3 @@ -component extends="ApplicationProxy" { +component extends="../ApplicationProxy" { // REST cfcs only look for an application.cfc in the same directory } \ No newline at end of file diff --git a/apps/updateserver/rest/ExtensionProvider.cfc b/apps/updateserver/rest/ExtensionProvider.cfc index 9f13fbc..5044e49 100644 --- a/apps/updateserver/rest/ExtensionProvider.cfc +++ b/apps/updateserver/rest/ExtensionProvider.cfc @@ -4,8 +4,63 @@ */ component { - variables.metaReader = application.extMetaReader; - variables.cdnURL = application.extensionsCdnUrl; + static { + static.DEBUG = (server.system.environment.DEBUG ?: false); + } + + variables.providerLog = "extension-provider"; + + private function getMetaReader() { + return application.extMetaReader; + } + + private function getCdnUrl() { + return application.extensionsCdnUrl; + } + + private function logger( string text, any exception, type="info", boolean forceSentry=false ){ + // var log = arguments.text & chr(13) & chr(10) & callstackGet('string'); + if ( !isNull(arguments.exception ) ){ + if (static.DEBUG) { + if ( len(arguments.text ) ) systemOutput( arguments.text, true ); + systemOutput( arguments.exception, true ); + } else { + writeLog( text=arguments.text, type=arguments.type, log="exception", exception=arguments.exception ); + // Send errors and warnings to Sentry (case insensitive check) + var normalizedType = lCase( arguments.type ); + if ( normalizedType == "error" || normalizedType == "warning" || normalizedType == "warn" ) { + try { + var sentryExtra = {}; + // Include custom text as context if provided + if ( len( arguments.text ) ) { + sentryExtra[ "logText" ] = arguments.text; + } + application.sentryLogger.logException( + exception = arguments.exception, + level = arguments.type, + extra = sentryExtra + ); + } catch ( any e ) { + // Don't let Sentry failures break anything + } + } + } + } else { + if (static.DEBUG) { + systemOutput( arguments.text, true); + } else { + writeLog( text=arguments.text, type=arguments.type, log=variables.providerLog ); + // Send to Sentry if forceSentry is true + if ( arguments.forceSentry ) { + try { + application.sentryLogger.logMessage( message=arguments.text, level=arguments.type ); + } catch ( any e ) { + // Don't let Sentry failures break anything + } + } + } + } + } /** * @httpmethod GET @@ -27,13 +82,14 @@ component { } }; try { - retVal.extensions = metaReader.list( + retVal.extensions = getMetaReader().list( type = ReFindNoCase( "^beta\.", hostName ) ? "abc" : arguments.type , flush = arguments.flush , withLogo = arguments.withLogo , coreVersion = coreVersion ); } catch( e ) { + logger(exception=e, type="error"); return e; } @@ -51,7 +107,7 @@ component { , string coreVersion = "" restargsource="url" , boolean flush = false restargsource="url" ){ - var ext = metaReader.getExtensionDetail( + var ext = getMetaReader().getExtensionDetail( id = arguments.id , withLogo = arguments.withLogo , version = arguments.version @@ -89,7 +145,7 @@ component { , boolean flush = false restargsource="url" ) { - var ext = metaReader.getExtensionDetail( + var ext = getMetaReader().getExtensionDetail( id = arguments.id , version = arguments.version , coreVersion = arguments.coreVersion @@ -99,7 +155,7 @@ component { if ( StructCount( ext ) ) { header statuscode="302" statustext="Found"; - header name="Location" value=variables.cdnURL & ext.filename; + header name="Location" value=getCdnUrl() & ext.filename; return; } @@ -119,7 +175,7 @@ component { , boolean flush = false restargsource="url" ) { - var ext = metaReader.getExtensionDetail( + var ext = getMetaReader().getExtensionDetail( id = arguments.id , version = arguments.version , coreVersion = arguments.coreVersion @@ -129,7 +185,7 @@ component { if ( StructCount( ext ) ) { header statuscode="302" statustext="Found"; - header name="Location" value=variables.cdnURL & ext.filename; + header name="Location" value=getCdnUrl() & ext.filename; return; } @@ -147,9 +203,10 @@ component { try { return { meta = {} - , extensions = metaReader.loadMeta() + , extensions = getMetaReader().loadMeta() } } catch( any e ) { + logger(exception=e, type="error"); return e; } } @@ -161,6 +218,6 @@ component { * @restPath reset */ remote function reset() { - metaReader.loadMeta(); + getMetaReader().loadMeta(); } } \ No newline at end of file diff --git a/apps/updateserver/rest/UpdateProvider.cfc b/apps/updateserver/rest/UpdateProvider.cfc index 9c5eb84..6205c52 100644 --- a/apps/updateserver/rest/UpdateProvider.cfc +++ b/apps/updateserver/rest/UpdateProvider.cfc @@ -3,28 +3,60 @@ * @restPath /update/provider */ component { - + static { + static.DEBUG = (server.system.environment.DEBUG ?: false); + } variables.bundleDownloadService = application.bundleDownloadService; variables.s3Root = application.coreS3Root; variables.cdnUrl = application.coreCdnUrl; variables.jiraChangelogService = application.jiraChangelogService; variables.providerLog = "update-provider"; + variables.LUCEE_MAVEN_CDN = "https://cdn.lucee.org/org/lucee/lucee"; ALL_VERSION="0.0.0.0"; MIN_UPDATE_VERSION="5.0.0.254"; MIN_NEW_CHANGELOG_VERSION="5.3.0.0"; MIN_WIN_UPDATE_VERSION="5.0.1.27"; - private function logger( string text, any exception, type="info" ){ - var log = arguments.text & chr(13) & chr(10) & callstackGet('string'); - if ( !isNull(arguments.exception ) ) - WriteLog( text=log, type=arguments.type, log=variables.providerLog, exception=arguments.exception ); - else - WriteLog( text=log, type=arguments.type, log=variables.providerLog ); + private function logger( string text, any exception, type="info", boolean forceSentry=false ){ + // var log = arguments.text & chr(13) & chr(10) & callstackGet('string'); + if ( !isNull(arguments.exception ) ){ + if (static.DEBUG) { + if ( len(arguments.text ) ) systemOutput( arguments.text, true ); + systemOutput( arguments.exception, true ); + } else { + WriteLog( text=arguments.text, type=arguments.type, log="exception", exception=arguments.exception ); + // Send errors and warnings to Sentry (case insensitive check) + var normalizedType = lCase( arguments.type ); + if ( normalizedType == "error" || normalizedType == "warning" || normalizedType == "warn" ) { + try { + application.sentryLogger.logException( exception=arguments.exception, level=arguments.type ); + } catch ( any e ) { + // Don't let Sentry failures break anything + } + } + } + } else { + if (static.DEBUG) { + systemOutput( arguments.text, true); + } else { + WriteLog( text=arguments.text, type=arguments.type, log=variables.providerLog ); + // Send to Sentry if forceSentry is true + if ( arguments.forceSentry ) { + try { + application.sentryLogger.logMessage( message=arguments.text, level=arguments.type ); + } catch ( any e ) { + // Don't let Sentry failures break anything + } + } + } + } } + /** + * MARK: /info * if there is a update the function is returning a struct like this: * {"type":"info" * ,"language":arguments.language @@ -41,160 +73,254 @@ component { * @langauage language f the requesting user */ remote struct function getInfo( - required string version restargsource="Path", - string ioid="" restargsource="url", - string language="en" restargsource="url") + required string version restargsource="Path", + string ioid="" restargsource="url", + string language="en" restargsource="url", + string extended=true restargsource="url") httpmethod="GET" restpath="info/{version}" { + try { + var s3 = new services.legacy.S3(variables.s3Root); + var rawList = s3.getVersions(); + var version = services.VersionUtils::toVersion(arguments.version); + var list = []; + loop array=rawList index="local.el" { + arrayAppend(list,local.el.version); + } + var data = s3.getLuceeVersionsDetail( version.display ); + var index = arrayFindNoCase( list, version.display ); + var latest = list[len(list)]; + var len = arrayLen( list ); + arrayDeleteAt( list,index ); + var rtn= { + "type":"info" + ,"language":arguments.language + ,"current":version.display + ,"latest":latest + ,"version": data.version?:"" + + ,"lastModified": data.lastModified?:"" + ,"size": data.size?:"0" + ,"etag": data.etag?:"" + ,"lco":data.lco?:createArtifactURL("lco",data.version) + ,"jar":data.jar + ,"light":data.light?:createArtifactURL("light",data.version) + ,"zero":data.zero?:createArtifactURL("zero",data.version) + ,"express":data.express?:createArtifactURL("express",data.version) + ,"war":data.war?:createArtifactURL("war",data.version) + ,"fb":data.forgebox?:createArtifactURL("fb",data.version) + ,"fbl":data["forgebox-light"]?:createArtifactURL("fbl",data.version) + }; + + + if(arguments.extended) { + rtn["otherVersions"]=list; + } + // we have an update? + if(len>index) { + rtn["available"]=latest; + rtn["message"]="A patch (#latest#) is available for your current version (#version.display#)."; + if(arguments.extended) rtn["changelog"]=getChangeLogs(version, services.VersionUtils::toVersion(latest)); + } + else { + rtn["message"]="There is no update available for your version (#version.display#). Latest version is [#latest#]." + } + return rtn; + } + catch(e){ + logger( text=e.message, exception=e, type="error" ); + return {"type":"error","message":e.message,cfcatch:e}; + } + } - try{ - local.version=toVersion(arguments.version); - - // no updates for versions smaller than ... - if(ALL_VERSION!=version.display && !isNewer(version,toVersion(MIN_UPDATE_VERSION))) - return { - "type":"warning", - "message":"Version ["&version.display&"] can not be updated from within the Lucee Administrator. Please update Lucee by replacing the lucee.jar, which can be downloaded from [http://download.lucee.org]"}; + /* + MARK: /download + */ + remote function downloadCore( + required string version restargsource="Path", + string ioid="" restargsource="url") + httpmethod="GET,HEAD" restpath="download/{version}" { + artifactDownloader( "lco", version ); + } + /** + * MARK: /core + */ + remote function downloadCoreAlias( + required string version restargsource="Path", + string ioid="" restargsource="url") + httpmethod="GET,HEAD" restpath="core/{version}" { - local.s3=new services.legacy.S3(variables.s3Root); - var versions=s3.getVersions(); - var keys=structKeyArray(versions); - arraySort(keys,"textnocase"); - var latest = {}; - latest.version = versions[keys[arrayLen(keys)]].version; - var latestVersion=toVersion(latest.version); + artifactDownloader( "lco", version ); + } - // others - latest.otherVersions=[]; - var maxSnap=400; - var maxRel=100; - if(ALL_VERSION!=version.display) { - for(var i=arrayLen(keys);i>=1;i--) { - var el=versions[keys[i]]; - if(findNoCase("-SNAPSHOT",el.version)) { - if ( ( --maxSnap )<=0 ) continue; - } - else { - if ( ( --maxRel )<=0 ) continue; - } - arrayPrepend(latest.otherVersions,el.version); - } - } + /** + * MARK: /lco + */ + remote function downloadLco( + required string version restargsource="Path", + string ioid="" restargsource="url") + httpmethod="GET,HEAD" restpath="lco/{version}" { - // no update - if(ALL_VERSION!=version.display && !isNewer(latestVersion,version)) - return { - "type":"info", - "message":"There is no update available for your version (#version.display#). Latest version is [#latestVersion.display#].", - "otherVersions":latest.otherVersions?:[] - }; + artifactDownloader( "lco", version ); + } - try { - var newChangeLog=isNewer(version,toVersion(MIN_NEW_CHANGELOG_VERSION)); - local.notes=(ALL_VERSION==version.display)? - "":getChangeLog(version.display,latestVersion.display); - - // do we need old layout of changelog? - if(!isNewer(version,toVersion(MIN_NEW_CHANGELOG_VERSION))) { - var nn=structNew("linked"); - loop struct=notes index="local.ver" item="local.dat" { - loop struct=dat index="local.k" item="local.v"{ - nn[k]=v; - } - } - notes=nn; - } - } - catch(local.ee){ - local.notes=""; - } + /** + * MARK: /loader + */ + remote function downloadLoader( + required string version restargsource="Path", + string ioid="" restargsource="url") + httpmethod="GET,HEAD" restpath="loader/{version}" { - var msgAppendix=""; - if(ALL_VERSION!=version.display && !isNewer(version,toVersion(MIN_WIN_UPDATE_VERSION))) - msgAppendix=" -
    Warning!
    - If this Lucee install is on a Windows based computer/server, please do not use the updater for this version due to a bug. Instead download the latest lucee.jar from     here and replace your existing lucee.jar with it. This is a one-time workaround."; + artifactDownloader( "loader", version ); + } + + /** + * only for backward compatibility + */ + remote function downLoaderAll( + required string version restargsource="Path", + string ioid="" restargsource="url") + httpmethod="GET,HEAD" restpath="loader-all/{version}" { + artifactDownloader( "loader", version ); + } + /** + * MARK: /jar + */ + remote function downloadJar( + required string version restargsource="Path", + string ioid="" restargsource="url") + httpmethod="GET,HEAD" restpath="jar/{version}" { - return { - "type":"info" - ,"language":arguments.language - ,"current":version.display - ,"available":latestVersion.display - ,"otherVersions":latest.otherVersions?:[] - ,"message":"A patch (#latestVersion.display#) is available for your current version (#version.display#)."&msgAppendix - ,"changelog":isSimpleValue(notes)?{}:notes/*readChangeLog(newest.log)*/ - }; // TODO get the right version for given version - } - catch(e){ - logger( text=e.message, exception=e, type="error" ); - return {"type":"error","message":e.message,cfcatch:e}; - } + artifactDownloader( "loader", version ); } + /* + MARK: /express + */ + remote function downloadExpress( + required string version restargsource="Path", + string ioid="" restargsource="url") + httpmethod="GET,HEAD" restpath="express/{version}" { + artifactDownloader( "express", version ); + } /** - * function to download Lucee Loader file (lucee.jar) - * return the download as a binary (application/zip), if there is no download available, the functions throws a exception + * MARK: /light */ - remote function downLoader( + remote function downloadLight( required string version restargsource="Path", string ioid="" restargsource="url") - httpmethod="GET" restpath="loader/{version}" { + httpmethod="GET,HEAD" restpath="light/{version}" { - createArtifactIfNecessary("jar",version); + artifactDownloader( "light", version ); + } - header statuscode="302" statustext="Found"; - header name="Location" value=variables.cdnURL&"lucee-"&arguments.version&".jar"; - return; + /** + * MARK: /zero + */ + remote function downloadZero( + required string version restargsource="Path", + string ioid="" restargsource="url") + httpmethod="GET,HEAD" restpath="zero/{version}" { + + artifactDownloader( "zero", version ); } /** - * function to download Light Lucee Loader file (lucee-light.jar) - * return the download as a binary (application/zip), if there is no download available, the functions throws a exception + * MARK: /war */ - remote function downLight( - required string version restargsource="Path", - string ioid="" restargsource="url") - httpmethod="GET" restpath="light/{version}" { - createArtifactIfNecessary("light",version); + remote function downloadWar( + required string version restargsource="Path", + string ioid="" restargsource="url") + httpmethod="GET,HEAD" restpath="war/{version}" { - header statuscode="302" statustext="Found"; - header name="Location" value=variables.cdnURL&"lucee-light-"&arguments.version&".jar"; - return; + artifactDownloader( "war", version ); } - /** - * only for backward compatibility + /* + MARK: /forgebox */ - remote function downLoaderAll( - required string version restargsource="Path", - string ioid="" restargsource="url") - httpmethod="GET" restpath="loader-all/{version}" { - return downLoaderNew(version,ioid); + remote function downloadForgebox( + required string version restargsource="Path", + string ioid="" restargsource="url", + boolean light=false restargsource="url") + httpmethod="GET,HEAD" restpath="forgebox/{version}" { + + artifactDownloader( light?"forgebox-light":"forgebox", version ); } - /** - * only for backward compatibility + /* + MARK: /fb */ - remote function downloadCoreAlias( - required string version restargsource="Path", - string ioid="" restargsource="url") - httpmethod="GET" restpath="core/{version}" { - return downloadCoreNew(version,ioid); + remote function downloadForgeboxAlias( + required string version restargsource="Path", + string ioid="" restargsource="url", + boolean light=false restargsource="url") + httpmethod="GET,HEAD" restpath="fb/{version}" { + + artifactDownloader( "forgebox", version ); + } + + /* + MARK: /fbl + */ + remote function downloadForgeboxLight( + required string version restargsource="Path", + string ioid="" restargsource="url") + httpmethod="GET,HEAD" restpath="fbl/{version}" { + + artifactDownloader( "forgebox-light", version ); + } + + /* + MARK: /localDevRepo + only for local development when not using a s3 bucket + */ + remote function downloadLocalDevRepo( + required string mavenPath restargsource="url" + ) + httpmethod="GET,HEAD" restpath="localDevRepo/" { + + if ( left( application.coreS3Root, 3 ) == "s3:" || mavenPath contains ".."){ + header statuscode=403; + echo("access denied"); + return; + } + var localMavenPath = application.coreS3Root & arguments.mavenPath; + if ( expandPath( application.coreS3Root & arguments.mavenPath) neq localMavenPath ){ + logger("bad localMavenPath: #localMavenPath#"); + header statuscode=403; + echo("access denied"); + return; + } + if ( fileExists( localMavenPath ) ) { + header name="Content-disposition" value="attachment;filename=#listlast(localMavenPath,'\/')#"; + content file="#localMavenPath#" type="application/octet-stream"; + } else { + logger("localMavenPath not found: #localMavenPath#"); + header statuscode=404; + echo("file not found"); + return; + } } + /** + * MARK: /echo + * echo functions are used by the lucee test suite + */ remote function echoGET( - string statusCode="" restargsource="url", - string mimeType="" restargsource="url", - string charset="" restargsource="url") - httpmethod="GET" - restpath="echoGet" { + string statusCode="" restargsource="url", + string mimeType="" restargsource="url", + string charset="" restargsource="url") + httpmethod="GET" restpath="echoGet" { + return _echo(arguments.statusCode, arguments.mimeType, arguments.charset); } remote function echoPOST() httpmethod="POST" restpath="echoPost" {return _echo();} @@ -228,56 +354,8 @@ component { return sct; } - remote function downloadCore( - required string version restargsource="Path", - string ioid="" restargsource="url") - httpmethod="GET" restpath="download/{version}" { - - createArtifactIfNecessary("lco",version); - - header statuscode="302" statustext="Found"; - header name="Location" value=variables.cdnURL&arguments.version&".lco"; - return; - } - - - - /** - * function to download Lucee Core file - * return the download as a binary (application/zip), if there is no download available, the functions throws a exception - */ - remote function downloadWarHead(required string version restargsource="Path", string ioid="" restargsource="url") - httpmethod="HEAD" restpath="war/{version}" { - return downloadWar(version, ioid); - } - - remote function downloadWar( - required string version restargsource="Path", - string ioid="" restargsource="url") - httpmethod="GET" restpath="war/{version}" { - - createArtifactIfNecessary("war",version); - - header statuscode="302" statustext="Found"; - header name="Location" value=variables.cdnURL&"lucee-"&arguments.version&".war"; - return; - } - - - remote function downloadForgebox( - required string version restargsource="Path", - string ioid="" restargsource="url", - boolean light=false restargsource="url") - httpmethod="GET" restpath="forgebox/{version}" { - - createArtifactIfNecessary(light?"fbl":"fb",version); - - header statuscode="302" statustext="Found"; - header name="Location" value=variables.cdnURL&"forgebox-"&(light?"light-":"")&arguments.version&".zip"; - return; - } - /** + * MARK: /download/{bundlename}/{bundleversion} * function to load 3rd party Bundle file, for example "/antlr/2.7.6" * relocate to a download URL or directly serve the download as a binary (application/zip). * @@ -345,41 +423,45 @@ component { httpmethod="GET" restpath="update-for/{version}" produces="application/lazy" { local.info=getInfo(version,ioid,"en"); - systemOutput(version&"-> "&structkeyExists(info,"available"),true,true); - + logger(version&"-> "&structkeyExists(info,"available")); if(structkeyExists(info,"available")) return info.available; return ""; } + /* + * MARK: /changelog + */ remote struct function getChangeLog( - required string versionFrom restargsource="Path", - required string versionTo restargsource="Path") + required string versionFrom restargsource="Path", + required string versionTo restargsource="Path") httpmethod="GET" restpath="changelog/{versionFrom}/{versionTo}" { return jiraChangelogService.getChangeLog( versionFrom=arguments.versionFrom, versionTo=arguments.versionTo ); } remote struct function getChangeLogExtended( - required string versionFrom restargsource="Path", - required string versionTo restargsource="Path") + required string versionFrom restargsource="Path", + required string versionTo restargsource="Path") httpmethod="GET" restpath="changelogDetailed/{versionFrom}/{versionTo}" { return jiraChangelogService.getChangeLog( versionFrom=arguments.versionFrom, versionTo=arguments.versionTo, detailed=true ); } remote string function getChangeLogLastUpdated() httpmethod="GET" restpath="changelogLastUpdated" { - // systemOutput("jiraChangelogService.getChangeLogUpdated():" & jiraChangelogService.getChangeLogUpdated(), true); + // logger("jiraChangelogService.getChangeLogUpdated():" & jiraChangelogService.getChangeLogUpdated()); + if (isNull(jiraChangelogService.getChangeLogUpdated())) return now(); return DateTimeFormat(jiraChangelogService.getChangeLogUpdated()); } /** - * function to get all dependencies (bundles) for a specific version - * @version version to get bundles for + * MARK: /dependencies + * function to get all dependencies (bundles) for a specific version + * @version version to get bundles for */ remote function downloadDependencies( - required string version restargsource="Path", - string ioid="" restargsource="url") + required string version restargsource="Path", + string ioid="" restargsource="url") httpmethod="GET" restpath="dependencies/{version}" { setting requesttimeout="1000"; @@ -388,6 +470,7 @@ component { local.path=mr.getDependencies(version); } catch(e){ + logger( exception=e, type="error" ); //return e; return {"type":"error","message":"The version #version# is not available."}; } @@ -402,8 +485,8 @@ component { * @version version to get bundles for */ remote function getDependencies( - required string version restargsource="Path", - string ioid="" restargsource="url") + required string version restargsource="Path", + string ioid="" restargsource="url") httpmethod="GET" restpath="dependencies-read/{version}" { setting requesttimeout="1000"; @@ -412,11 +495,17 @@ component { return mr.getOSGiDependencies(version,true); } catch(e){ + logger( exception=e, type="error" ); return {"type":"error","message":"The version #version# is not available."}; } } - remote function getExpressTemplates() httpmethod="GET" restpath="expressTemplates" { + /* + MARK: /expressTemplates + */ + + remote function getExpressTemplates() + httpmethod="GET" restpath="expressTemplates" { var s3 = new services.legacy.S3( variables.s3Root ); var expressTemplates = duplicate( s3.getExpressTemplates() ); loop collection=#expressTemplates# key="local.key" item="local.item"{ @@ -432,6 +521,10 @@ component { jiraChangelogService.updateIssuesAsync(); // async } + /* + MARK: /latest + */ + remote function getLatest( string version restargsource="path", string type restargsource="path", // stable rc beta snapshot all @@ -440,16 +533,14 @@ component { ) httpmethod="GET" restpath="latest/{version}/{type}/{distribution}/{format}" { try { - var s3 = new services.legacy.S3( variables.s3Root ); - var versions = s3.getVersions(); - + var s3 = new services.legacy.S3(variables.s3Root); + var versions=services.VersionUtils::versionArrayToStruct(s3.getVersions()); if ( arguments.type eq "all" ) arguments.type =""; if ( arguments.version eq 0 ) arguments.version = ""; - - var matchedVersion = services.VersionUtils::matchVersion( versions, arguments.type, + var matchedVersion = services.VersionUtils::matchVersion( versions, arguments.type, arguments.version, arguments.distribution ); // i.e. 06.002.001.0048.000 if ( len( matchedVersion ) eq 0 ){ @@ -458,7 +549,7 @@ component { } if ( len( arguments.format ) eq 0) arguments.format = "redirect"; - + var _version = versions[ matchedVersion ].version; // i.e 6.2.1.48-SNAPSHOT @@ -487,77 +578,85 @@ component { } } catch(e){ - systemOutput( e, 1, 1 ); header statuscode="500"; logger(text=e.message, exception=e, type="error"); echo (e.message); } } + /* + MARK: /list + */ remote function readList( - boolean force=false restargsource="url", - string type='all' restargsource="url", - boolean extended=false restargsource="url", - boolean flush=false restargsource="url" + boolean force=false restargsource="url", + string type='all' restargsource="url", + boolean extended=false restargsource="url", + boolean flush=false restargsource="url" ) httpmethod="GET" restpath="list" { - setting requesttimeout="1000"; - - try { - - var s3=new services.legacy.S3(variables.s3Root); - var versions=s3.getVersions(flush); + var rtn=arguments.extended?[:]:[]; + var ignores=["6.0.0.12-SNAPSHOT","6.0.0.13-SNAPSHOT","6.0.1.82","7.0.0.202"]; + var s3 = new services.legacy.S3(variables.s3Root); + var versions=s3.getVersions( flush ); - if(!isNull(url.abc)){ - return versions; - } + // when working locally, route cdn requests thru /localDevRepo + var localMaven = (left( application.coreS3Root, 3 ) != "s3:"); - var ignores=["6.0.0.12-SNAPSHOT","6.0.0.13-SNAPSHOT","6.0.1.82"]; - loop array=structKeyArray( versions ) item="local.k" { - if ( !structKeyExists( versions[ k ], "version" ) - || arrayFind( ignores, versions[k].version ) ){ - structDelete( versions, k ); + loop array=versions index="local.el" { + try { + if(arrayContainsNoCase(ignores,el.version)) continue; + local.sct=services.VersionUtils::toVersion(el.version); + if(!arguments.extended) { + arrayAppend(rtn, + { + "version":sct.display, + "vs":sct.sortable + }); + } + else { + if (localMaven) el = rewriteLocalMaven(el); + + rtn[local.sct.sortable]={ + "version": sct.display, + "lastModified": el.lastModified?:"", + "size": el.size?:"0", + "etag": el.etag?:"", + "lco":el.lco ?: createArtifactURL("lco",sct.display), + "jar":el.jar, + "light":el.light?:createArtifactURL("light",sct.display), + "zero":el.zero?:createArtifactURL("zero",sct.display), + "express":el.express?:createArtifactURL("express",sct.display), + "war":el.war?:createArtifactURL("war",sct.display), + "fb":el.forgebox?:createArtifactURL("fb",sct.display), + "fbl":el["forgebox-light"]?:createArtifactURL("fbl",sct.display) + }; } } - - if ( extended ) return versions; - var arr=[]; - loop struct=versions index="local.vs" item="local.data" { - arrayAppend(arr,{'vs':vs,'version':data.version}); + catch (any e) { + logger( exception=e, type="error" ); } - return arr; - } - catch(e){ - systemOutput( e, 1, 1 ); - logger( error=e.message, exception=e, type="error" ); - return {"type":"error","message":e.message}; } + return rtn; } + + remote function getDate(required string version restargsource="Path") httpmethod="GET" restpath="getdate/{version}" { - var mr=new services.legacy.MavenRepo(); - try{ - var info=mr.get(version,true); - if(!isNull(info.sources.pom.date)) - return parseDateTime(info.sources.pom.date); - else if(!isNull(info.sources.jar.date)) - return parseDateTime(info.sources.jar.date); - } catch(e) { - //systemOutput(e.stackTrace, true); - var mess= "maven.getDate() threw " & left(cfcatch.message,100); - logger(text=mess, type="error"); - systemOutput(mess, true, true ); - } + var s3 = new services.legacy.S3(variables.s3Root); + var detail=s3.getLuceeVersionsDetail(version); - return ""; + // TODO get data from LuceeVersionsDetail, make it availabe there + //if(static.DEBUG) systemOutput(LuceeVersionsDetail(version), true, true); + + return detail.lastModified?:""; } remote function readGetOnlyForDebugging( - required string version restargsource="Path" - ,boolean extended restargsource="url") + required string version restargsource="Path" + ,boolean extended restargsource="url") httpmethod="GET" restpath="get/{version}" { setting requesttimeout="1000"; @@ -566,31 +665,20 @@ component { return mr.get(arguments.version,arguments.extended); } catch(e){ + logger( exception=e, type="error" ); return {"type":"error","message":e.message}; } } - remote function downloadExpress( - required string version restargsource="Path", - string ioid="" restargsource="url") - httpmethod="GET" restpath="express/{version}" { - - createArtifactIfNecessary("express",version); - - header statuscode="302" statustext="Found"; - header name="Location" value=variables.cdnURL&"lucee-express-"&arguments.version&".zip"; - return; - } - - remote function listMissing(boolean inclFB=false restargsource="url") + remote function listMissing( + boolean inclFB=false restargsource="url") httpmethod="GET" restpath="list-missing" { setting requesttimeout="100"; - var s3=new services.legacy.S3(variables.s3Root); - var versions=s3.getVersions(true); - + var s3 = new services.legacy.S3(variables.s3Root); + var versions=services.VersionUtils::versionArrayToStruct(s3.getVersions()); var rtn=structNew("linked"); var list="jar,lco,war,light,express"; - if(inclFB)list&=",fb,fbl"; + if(inclFB)list&=",forgebox,forgebox-light"; loop list=list item="type" { loop struct=versions index="vs" item="data" { @@ -604,6 +692,7 @@ component { } /** + * MARK: /buildLatest * this functions triggers that everything is prepared/build for future requests * @version version to get bundles for */ @@ -613,98 +702,11 @@ component { var indexDir=getDirectoryFromPath(getCurrentTemplatePath())&"index/"; if(directoryExists(indexDir)) directoryDelete(indexDir, true); - - var s3=new services.legacy.S3(variables.s3Root); - s3.addMissing(true); + var s3=new services.legacy.S3(variables.s3Root); + s3.buildLatest(true); return "done"; } - private boolean function isVersion(required string version) { - try{ - toVersion(version); - return true; - } - catch(e) { - return false; - } - } - - private struct function toVersion(required string version, boolean ignoreInvalidVersion=false){ - local.arr=listToArray(arguments.version,'.'); - if(arr.len()==3) { - arr[4]="0"; - } - if(arr.len()!=4 || !isNumeric(arr[1]) || !isNumeric(arr[2]) || !isNumeric(arr[3])) { - if(ignoreInvalidVersion) return {}; - throw ("version number ["&arguments.version&"] is invalid"); - } - local.sct={major:arr[1]+0,minor:arr[2]+0,micro:arr[3]+0,qualifier_appendix:"",qualifier_appendix_nbr:100}; - - // qualifier has an appendix? (BETA,SNAPSHOT) - local.qArr=listToArray(arr[4],'-'); - if(qArr.len()==1 && isNumeric(qArr[1])) local.sct.qualifier=qArr[1]+0; - else if(qArr.len()==2 && isNumeric(qArr[1])) { - sct.qualifier=qArr[1]+0; - sct.qualifier_appendix=qArr[2]; - if(sct.qualifier_appendix=="SNAPSHOT")sct.qualifier_appendix_nbr=0; - else if(sct.qualifier_appendix=="BETA")sct.qualifier_appendix_nbr=50; - else sct.qualifier_appendix_nbr=75; // every other appendix is better than SNAPSHOT - } - else throw ("version number ["&arguments.version&"] is invalid"); - sct.pure= - sct.major - &"."&sct.minor - &"."&sct.micro - &"."&sct.qualifier; - sct.display= - sct.pure - &(sct.qualifier_appendix==""?"":"-"&sct.qualifier_appendix); - - sct.sortable=repeatString("0",2-len(sct.major))&sct.major - &"."&repeatString("0",3-len(sct.minor))&sct.minor - &"."&repeatString("0",3-len(sct.micro))&sct.micro - &"."&repeatString("0",4-len(sct.qualifier))&sct.qualifier - &"."&repeatString("0",3-len(sct.qualifier_appendix_nbr))&sct.qualifier_appendix_nbr; - return sct; - } - - private boolean function isNewer(required struct left, required struct right ){ - // major - if(left.major>right.major) return true; - if(left.majorright.minor) return true; - if(left.minorright.micro) return true; - if(left.microright.qualifier) return true; - if(left.qualifierright.qualifier_appendix_nbr) return true; - if(left.qualifier_appendix_nbrright.qualifier_appendix) return true; - if(left.qualifier_appendix"&(variables.s3Root&name)); - - var hasDef=(!isNull(application.exists[name]) && application.exists[name]); - if(hasDef || fileExists(variables.s3Root&name)) { - application.exists[name]=true; - header statuscode="302" statustext="Found"; - header name="Location" value=variables.cdnURL&name; - return true; - } - // if not exist we make ready for the next - else { - - if(async && isNull(url.show)) { - thread src=path trg=variables.s3Root&name { - lock timeout=1000 name=src { - if(!fileExists(trg) && fileSize(src)>100000) // we do this because it was created by a thread blocking this thread - _fileCopy(src,trg); - } - } - } - else { - var src=path; - var trg=variables.s3Root&name; - lock timeout=1000 name=src { - if(!fileExists(trg) && fileSize(src)>100000) {// we do this because it was created by a thread blocking this thread - _fileCopy(src,trg); - if(!isNull(url.show)) - throw ("fileExists: "&fileExists(src)&" + "&fileExists(trg)); - } - } - } - } - return false; - } - private function _fileCopy(src,trg) { if(isSimpleValue(src) && findNoCase("http",src)==1) { // we do this because of 302 the function cannot handle @@ -782,33 +741,120 @@ component { private function fileSize(path) { - var dir=getDirectoryFromPath(path); - var file=listLast(path,'\/'); - directory filter=file name="local.res" directory=dir action="list"; - return res.recordcount==1?res.size:0; + var dir=getDirectoryFromPath(path); + var file=listLast(path,'\/'); + directory filter=file name="local.res" directory=dir action="list"; + return res.recordcount==1?res.size:0; } - private function createArtifactIfNecessary(type,version) { - var s3=new services.legacy.S3(variables.s3Root); - var versions=s3.getVersions(); - var vs=services.VersionUtils::toVersionSortable(version); + /* + MARK: artifactDownloader + */ + + private function artifactDownloader( type, version ) { + var _url=createArtifactIfNecessary( type, version ); + header statuscode="302" statustext="Found"; + if (structKeyExists(local,"_url")) { + header name="Content-disposition" value="attachment;filename=#listlast( _url, '\/' )#"; + if (left( application.coreS3Root, 3 ) == "s3:"){ + header name="Location" value=_url; + } else { + // route thru /localDevRepo + header name="Location" value=rewriteLocalMaven( { _url } )._url; + } + } else { + // is this reachable, need to handle local dev repo? src is obviously wrong + header name="Location" value="#LUCEE_MAVEN_CDN#/#arguments.version#/lucee-#arguments.version#.#arguments.type#"; + } + } - if(structKeyExists(versions,vs) && structKeyExists(versions[vs],type)) return; - thread s3=s3 name=createUUID() _type=type _version=version { + /* + MARK: createArtifactIfNecessary + */ + + private function createArtifactIfNecessary( type , version ) { + var versionData=services.VersionUtils::toVersion( arguments.version ); + var s3=new services.legacy.S3( variables.s3Root ); + var data=s3.getLuceeVersionsDetail( versionData.display ); + + logger("createArtifactIfNecessary(#type#,#version#) ---"); + + // in case we have a link for it, no action is needed + if (!isNull(data[arguments.type])) { + logger("--- found a match: "&data[arguments.type]); + return data[arguments.type]; + } + logger("--- no match found, creating artifact"); + + var threadName="t"&createUUID(); + thread s3=s3 name=threadName _type=type _version=version { try{ - setting requesttimeout="10000000"; s3.add(_type,_version); } catch(e){ - fileWrite("error.txt",serialize(e)); + logger( exception=e, type="error" ); } } - sleep(20000); - versions=s3.getVersions(); - if(structKeyExists(versions,vs) && structKeyExists(versions[vs],type)) return; // all good, was built in the meantime + + // wait for the thread to finish + logger("Waiting for thread #threadName# to finish..."); + threadJoin(threadName,50000); + + // check if the artifact was created + var data=s3.getLuceeVersionsDetail(versionData.display); + + if(!isNull(data[arguments.type])) { + return data[arguments.type]; + } content type="text/plain"; header statuscode="429" statustext="Still Building"; echo("artifact #encodeForHtml(type)# for version #encodeForHtml(version)# does not exist yet, but we triggered the build for it. Try again in a couple minutes."); abort; } + + private function createArtifactURL(type,version) { + return "#getBaseURL()##arguments.type#/#arguments.version#"; + } + private function getBaseURL() { + return application.updateProviderUrl; + } + + private function getChangeLogs(struct version, struct latestVersion) { + try { + var newChangeLog=services.VersionUtils::isNewer(version, services.VersionUtils::toVersion(MIN_NEW_CHANGELOG_VERSION)); + local.notes=(ALL_VERSION==version.display)? + "":getChangeLog(version.display,latestVersion.display); + + // do we need old layout of changelog? + if(!services.VersionUtils::isNewer(version, services.VersionUtils::toVersion(MIN_NEW_CHANGELOG_VERSION))) { + var nn=structNew("linked"); + loop struct=notes index="local.ver" item="local.dat" { + loop struct=dat index="local.k" item="local.v"{ + nn[k]=v; + } + } + notes=nn; + } + } + catch(local.ee){ + logger( exception=ee, type="error" ); + local.notes=""; + } + return local.notes; + } + + private function rewriteLocalMaven( src ){ + var st = [=]; + var l = len ( application.coreS3Root ) + 1; + var baseUrl = getBaseURL(); + loop collection=#src# key="local.k" value="local.v" { + if ( left( v, 1 ) eq "/" ){ + st[k] = baseUrl & "localDevRepo?mavenPath=#mid(v,l)#"; + } else { + st[k] = v; + } + } + return st; + } + } \ No newline at end of file diff --git a/apps/updateserver/services/ExtensionMetadataReader.cfc b/apps/updateserver/services/ExtensionMetadataReader.cfc index d82ee74..7ef56c1 100644 --- a/apps/updateserver/services/ExtensionMetadataReader.cfc +++ b/apps/updateserver/services/ExtensionMetadataReader.cfc @@ -1,11 +1,16 @@ component accessors=true { + static { + static.DEBUG = (server.system.environment.DEBUG ?: false); + } + property name="s3root" type="string" default=""; property name="extensionMeta" type="query"; property name="extensionVersions" type="struct"; property name="bundleDownloadService" type="any"; reset(); + variables.providerLog = "update-provider"; function reset() { variables._simpleCache = StructNew( "max:100" ); @@ -13,20 +18,29 @@ component accessors=true { function loadMeta( query srcMeta ) { lock type="exclusive" name="readExtMeta" timeout=0 { - var meta = isQuery( arguments.srcMeta ) ? arguments.srcMeta : _readExistingMetaFileFromS3(); + logger( "Loading Extension metadata" ); + var meta = len( arguments.srcMeta ) ? arguments.srcMeta : _readExistingMetaFileFromS3(); var existingByFile = _mapExtensionQueryByFilename( meta ); var lexFiles = _listLexFilesFromBucket(); var metaChanged = false; if (meta.recordcount == 0){ // rebuilding the index takes a while as all extensions need to be re-downloaded and processed - setting requesttimeout="#lexFiles.recordcount#*4"; + setting requesttimeout="#(lexFiles.recordcount*4)#"; } for( var lexFile in lexFiles ) { var isNewToUs = !StructKeyExists( existingByFile, lexFile.name ) if ( isNewToUs ) { - _addLexFile( lexFile.name, meta ); + _addLexFile( lexFile, meta, false ); + metaChanged = true; + } else if ( len( existingByFile[ lexFile.name ].updated ) == 0 ) { + // tmp workaround, need to add metadata + _updateCreatedDate( meta, lexFile ); + metaChanged = true; + } else if ( dateCompare( lexFile.dateLastModified, existingByFile[ lexFile.name ].updated ) > 0 ) { + // ext was file was modified + _addLexFile( lexFile, meta, true ); metaChanged = true; } } @@ -184,6 +198,10 @@ component accessors=true { var metaFile = getS3Root() & "/extensions.json"; if ( FileExists( metaFile ) ) { var meta = DeserializeJson( FileRead( metaFile ), false ); + // tmp workaround, need to add metadata + if ( ! QueryColumnExists( meta, "updated" ) ) { + QueryAddColumn( meta, "updated", "date" ); + } return _updateExtensionSorting( meta ); } return _getEmptyExtensionsQuery(); @@ -211,7 +229,7 @@ component accessors=true { } private function _getEmptyExtensionsQuery() { - return QueryNew( "id,version,versionSortable,name,description,filename,image,category,author,created,releaseType," + return QueryNew( "id,version,versionSortable,name,description,filename,image,category,author,created,updated,releaseType," & "minLoaderVersion,minCoreVersion,price,currency,disableFull,trial,older,olderName," & "olderDate,promotionLevel,promotionText,projectUrl,sourceUrl,documentionUrl" ); } @@ -226,21 +244,32 @@ component accessors=true { return mapped; } - private function _addLexFile( fileName, extensionMetaQuery ) { - SystemOutput( "Loading new extension file [#arguments.fileName#] from S3.", true ); + private function _addLexFile( lexFile, extensionMetaQuery, boolean update=false ) { + logger( "Loading new extension file [#arguments.lexFile.name#] from S3, update=#arguments.update#." ); try { - var tmpFile = _copyRemoteFileToTmpFile( arguments.fileName ); + var tmpFile = _copyRemoteFileToTmpFile( arguments.lexFile.name ); var qryCols = ListToArray( arguments.extensionMetaQuery.columnList ); var extMeta = _initExtensionMetaFromManifest( tmpFile, qryCols ); - extMeta.filename = arguments.fileName; + extMeta.filename = arguments.lexFile.name; extMeta.versionSortable = Len( extMeta.version ) ? VersionUtils::sortableVersionString( extMeta.version ) : ""; extMeta.trial = false; // "TODO"??? extMeta.releaseType = Len( extMeta.releaseType ) ? extMeta.releaseType : "all"; extMeta.image = _getLogoThumbnail( tmpFile ); - - QueryAddRow( arguments.extensionMetaQuery, extMeta ); + extMeta.updated = arguments.lexFile.dateLastModified; + + if ( arguments.update ){ + var existingRow = _getRowNumberByFilename( arguments.extensionMetaQuery, arguments.lexFile.name ); + if ( existingRow ){ + logger( "Updated lex file metadata [#arguments.lexfile.name#]"); + QuerySetRow( arguments.extensionMetaQuery, existingRow, extMeta ); + } else { + logger( "Error updating lex file [#arguments.lexfile.name#], could not find in existing query ", type="error"); + } + } else { + QueryAddRow( arguments.extensionMetaQuery, extMeta ); + } _processExtensionJars( tmpFile ); @@ -251,14 +280,16 @@ component accessors=true { } } catch( any e ) { // for now, just to get through them all! - SystemOutput( "Error processing lex file: [#arguments.fileName#]", true ); + logger( text="Error processing lex file: [#arguments.lexFile.name#]",exception=e, type="error" ); } } private function _readLexManifest( filePath ) { try { var mf = ManifestRead( arguments.filePath ); - } catch( any e ) {} + } catch( any e ) { + logger( text="Error reading manifest [#arguments.filePath#]",exception=e, type="error" ); + } return mf.main ?: {}; } @@ -431,7 +462,7 @@ component accessors=true { } if ( !found ) { changed = true; - SystemOutput( "Deleting [#arguments.cachedExts.filename[ i ]#] extension from cached query as it no longer exists in our lex file lookup.", true ); + logger( "Deleting [#arguments.cachedExts.filename[ i ]#] extension from cached query as it no longer exists in our lex file lookup."); QueryDeleteRow( arguments.cachedExts, i ); } } @@ -459,4 +490,67 @@ component accessors=true { } throw "No extension version available for [#arguments.coreVersion#]"; }; + + private function _updateCreatedDate( extensionMetaQuery, lexFile) { + var existingRow = _getRowNumberByFilename( arguments.extensionMetaQuery, arguments.lexFile.name ); + if (existingRow > 0) { + QuerySetCell( arguments.extensionMetaQuery, "updated", arguments.lexFile.dateLastModified, existingRow ); + } else { + logger( "Error updating lex file [#arguments.lexfile.name#], could not find in existing query ", type="error"); + } + } + + private function _getRowNumberByFilename( extensionMetaQuery, filename ){ + for( var i=1; i <= arguments.extensionMetaQuery.recordcount; i++ ) { + if ( arguments.extensionMetaQuery.filename[ i ] == arguments.filename ) { + return i; + } + } + return -1; + } + + private function logger( string text, any exception, type="info", boolean forceSentry=false ){ + //var log = arguments.text & chr(13) & chr(10) & callstackGet('string'); + if ( !isNull(arguments.exception ) ){ + + if (static.DEBUG) { + if (len(arguments.text)) systemOutput( arguments.text, true, true ); + systemOutput( arguments.exception, true, true ); + } else { + writeLog( text=arguments.text, type=arguments.type, log="exception", exception=arguments.exception ); + // Send errors and warnings to Sentry (case insensitive check) + var normalizedType = lCase( arguments.type ); + if ( normalizedType == "error" || normalizedType == "warning" || normalizedType == "warn" ) { + try { + var sentryExtra = {}; + // Include custom text as context if provided + if ( len( arguments.text ) ) { + sentryExtra[ "logText" ] = arguments.text; + } + application.sentryLogger.logException( + exception = arguments.exception, + level = arguments.type, + extra = sentryExtra + ); + } catch ( any e ) { + // Don't let Sentry failures break anything + } + } + } + } else { + if (static.DEBUG) { + systemOutput( arguments.text, true, true ); + } else { + writeLog( text=arguments.text, type=arguments.type, log="application" ); + // Send to Sentry if forceSentry is true + if ( arguments.forceSentry ) { + try { + application.sentryLogger.logMessage( message=arguments.text, level=arguments.type ); + } catch ( any e ) { + // Don't let Sentry failures break anything + } + } + } + } + } } diff --git a/apps/updateserver/services/JiraChangeLogService.cfc b/apps/updateserver/services/JiraChangeLogService.cfc index efc628c..58e17e7 100644 --- a/apps/updateserver/services/JiraChangeLogService.cfc +++ b/apps/updateserver/services/JiraChangeLogService.cfc @@ -109,11 +109,11 @@ component accessors=true { } private function _fetchIssues() { - systemOutput("-- start fetching issues from jira --- ", true); + systemOutput( "[#dateTimeFormat(now(), "long")#] -- start fetching issues from jira --- ", true); var jira = new services.JiraCloud({ domain: getJiraServer() }); var issuesArray = jira.searchIssues(jql="project=LDEV AND status in (Deployed, Done, QA, Resolved)"); var issuesQuery = _issuesArrayToQuery(issuesArray); - systemOutput("-- finished fetching issues from jira --- ", true); + systemOutput( "[#dateTimeFormat(now(), "long")#] -- finished fetching issues from jira --- ", true); return issuesQuery; } @@ -150,6 +150,8 @@ component accessors=true { if ( FileExists( issues ) ) { systemOutput("jira._readExistingIssuesFromS3", true); return DeserializeJson( FileRead( issues ), false ); + } else { + systemOutput("jira._readExistingIssuesFromS3 - no existing issues file [#issues#]", true); } return _getEmptyIssuesQuery(); } diff --git a/apps/updateserver/services/SentryLogger.cfc b/apps/updateserver/services/SentryLogger.cfc new file mode 100644 index 0000000..b041f12 --- /dev/null +++ b/apps/updateserver/services/SentryLogger.cfc @@ -0,0 +1,375 @@ +/** + * A CFML component for logging events and errors to Sentry. + * Supports Sentry's Store API for sending events. + */ +component { + + /** + * Constructor for the SentryLogger component. + * + * @param config A struct containing the Sentry connection details. + * - dsn: Your Sentry DSN (e.g., "https://key@sentry.io/project-id"). + * - environment: The environment name (e.g., "production", "staging", "development"). Defaults to "production". + * - release: Optional release identifier. + * - serverName: Optional server name. Defaults to cgi.server_name. + */ + public function init( required struct config ) { + variables.config = arguments.config; + + // DSN is optional - if not provided or empty, logger will exercise all code paths but not send to Sentry + // Strip quotes in case DSN comes through as literal '""' string from environment variables + var cleanDsn = trim( replace( variables.config.dsn ?: "", '""', '', 'all' ) ); + if ( len( cleanDsn ) && cleanDsn neq '""' ) { + // Parse the DSN + variables.sentryInfo = _parseDsn( cleanDsn ); + variables.hasDsn = true; + } else { + variables.hasDsn = false; + } + + // Set defaults + if ( !structKeyExists( variables.config, "environment" ) || isEmpty( variables.config.environment ) ) { + variables.config.environment = "production"; + } + + if ( !structKeyExists( variables.config, "serverName" ) || isEmpty( variables.config.serverName ) ) { + variables.config.serverName = cgi.server_name ?: "unknown"; + } + + return this; + } + + /** + * Logs a message to Sentry. + * + * @param message The message to log. + * @param level The severity level: "debug", "info", "warning", "error", "fatal". Defaults to "info". + * @param extra Additional context data to send with the event. + * @param tags Tags to categorize the event. + * @param user User information (id, email, username, ip_address). + */ + public struct function logMessage( + required string message, + string level = "info", + struct extra = {}, + struct tags = {}, + struct user = {} + ) { + var event = _buildEvent( + message = arguments.message, + level = arguments.level, + extra = arguments.extra, + tags = arguments.tags, + user = arguments.user + ); + + return _sendEvent( event ); + } + + /** + * Logs an exception to Sentry. + * + * @param exception The exception object (cfcatch). + * @param level The severity level. Defaults to "error". + * @param extra Additional context data to send with the event. + * @param tags Tags to categorize the event. + * @param user User information. + */ + public struct function logException( + required any exception, + string level = "error", + struct extra = {}, + struct tags = {}, + struct user = {} + ) { + // Handle simple values (string, number, etc) passed as exception + var exceptionMessage = "Unknown error"; + var exceptionType = "Error"; + var exceptionStruct = arguments.exception; + + if ( !isStruct( arguments.exception ) ) { + // Simple value passed - convert to string + exceptionMessage = toString( arguments.exception ); + exceptionStruct = {}; + } else { + exceptionMessage = arguments.exception.message ?: toString( arguments.exception ); + exceptionType = arguments.exception.type ?: "Error"; + } + + // Use custom logText as the main message if provided, otherwise use exception message + var displayMessage = exceptionMessage; + if ( structKeyExists( arguments.extra, "logText" ) && len( arguments.extra.logText ) ) { + displayMessage = arguments.extra.logText; + // Keep the original exception message in extra context + arguments.extra[ "exceptionMessage" ] = exceptionMessage; + } + + var event = _buildEvent( + message = displayMessage, + level = arguments.level, + extra = arguments.extra, + tags = arguments.tags, + user = arguments.user + ); + + // Add exception details + event[ "exception" ] = { + "values" = [ + { + "type" = exceptionType, + "value" = exceptionMessage, + "stacktrace" = _parseStackTrace( exceptionStruct ) + } + ] + }; + + // Add additional exception context if it's a proper exception struct + if ( isStruct( arguments.exception ) ) { + if ( structKeyExists( arguments.exception, "detail" ) && len( arguments.exception.detail ) ) { + event.extra[ "detail" ] = arguments.exception.detail; + } + + if ( structKeyExists( arguments.exception, "extendedInfo" ) && len( arguments.exception.extendedInfo ) ) { + event.extra[ "extendedInfo" ] = arguments.exception.extendedInfo; + } + } + + return _sendEvent( event ); + } + + /** + * Builds a base event structure for Sentry. + */ + private struct function _buildEvent( + required string message, + required string level, + required struct extra, + required struct tags, + required struct user + ) { + var event = { + "event_id" = lCase( replace( createUUID(), "-", "", "all" ) ), + "timestamp" = _getIsoTimestamp(), + "level" = _normalizeSentryLevel( arguments.level ), + "message" = arguments.message, + "platform" = "cfml", + "environment" = variables.config.environment, + "server_name" = variables.config.serverName, + "extra" = arguments.extra, + "tags" = arguments.tags + }; + + // Add release if configured + if ( structKeyExists( variables.config, "release" ) && len( variables.config.release ) ) { + event[ "release" ] = variables.config.release; + } + + // Add user context if provided + if ( structCount( arguments.user ) ) { + event[ "user" ] = arguments.user; + } + + // Add request context + event[ "request" ] = _getRequestContext(); + + // Add server context + event[ "contexts" ] = _getServerContext(); + + return event; + } + + /** + * Sends an event to Sentry via HTTP. + */ + private struct function _sendEvent( required struct event ) { + // If no DSN configured, skip the HTTP call but return success + // This allows full code path testing in dev without sending to Sentry + if ( !variables.hasDsn ) { + return { + "success" = true, + "eventId" = arguments.event.event_id, + "statusCode" = "200", + "note" = "No DSN configured - event not sent to Sentry" + }; + } + + var fullUrl = variables.sentryInfo.apiUrl; + var timestamp = _getTimestamp(); + + try { + cfhttp( method="POST", url=fullUrl, result="local.result", throwOnError=false, timeout=5 ) { + cfhttpparam( type="header", name="Content-Type", value="application/json" ); + cfhttpparam( type="header", name="X-Sentry-Auth", value=_buildAuthHeader( timestamp ) ); + cfhttpparam( type="body", value=serializeJson( arguments.event ) ); + } + + if ( !find( "200", local.result.statusCode ) ) { + throw( + type = "Sentry.HTTPException", + message = "Sentry API returned status: #local.result.statusCode#", + detail = "Response: " & local.result.fileContent + ); + } + + return { + "success" = true, + "eventId" = arguments.event.event_id, + "statusCode" = local.result.statusCode + }; + + } catch ( any e ) { + // Don't let logging failures break the application + // You could log this to a file or dump it + return { + "success" = false, + "error" = e.message, + "detail" = e.detail ?: "" + }; + } + } + + /** + * Parses a Sentry DSN into its components. + */ + private struct function _parseDsn( required string dsn ) { + // DSN format: https://{publicKey}@{host}/{projectId} + var pattern = "^(https?)://([^@]+)@([^/]+)/(.+)$"; + var matches = reFind( pattern, arguments.dsn, 1, true ); + + if ( !matches.pos[ 1 ] ) { + throw( type="Sentry.ConfigurationException", message="Invalid DSN format" ); + } + + var protocol = mid( arguments.dsn, matches.pos[ 2 ], matches.len[ 2 ] ); + var publicKey = mid( arguments.dsn, matches.pos[ 3 ], matches.len[ 3 ] ); + var host = mid( arguments.dsn, matches.pos[ 4 ], matches.len[ 4 ] ); + var projectId = mid( arguments.dsn, matches.pos[ 5 ], matches.len[ 5 ] ); + + return { + "protocol" = protocol, + "publicKey" = publicKey, + "host" = host, + "projectId" = projectId, + "apiUrl" = "#protocol#://#host#/api/#projectId#/store/" + }; + } + + /** + * Builds the Sentry auth header. + */ + private string function _buildAuthHeader( required numeric timestamp ) { + var parts = [ + "Sentry sentry_version=7", + "sentry_client=cfml-sentry/1.0", + "sentry_timestamp=#arguments.timestamp#", + "sentry_key=#variables.sentryInfo.publicKey#" + ]; + + return arrayToList( parts, ", " ); + } + + /** + * Gets the current timestamp in seconds. + */ + private numeric function _getTimestamp() { + return fix( getTickCount() / 1000 ); + } + + /** + * Gets the current timestamp in ISO 8601 format. + */ + private string function _getIsoTimestamp() { + return dateTimeFormat( now(), "iso8601" ); + } + + /** + * Extracts request context from CGI scope. + */ + private struct function _getRequestContext() { + var context = { + "url" = cgi.server_name & cgi.script_name, + "method" = cgi.request_method, + "query_string" = cgi.query_string, + "headers" = {} + }; + + // Add common headers including user agent + var headerKeys = [ "user-agent", "referer", "content-type" ]; + for ( var key in headerKeys ) { + var cgiKey = "http_" & replace( key, "-", "_", "all" ); + if ( structKeyExists( cgi, cgiKey ) ) { + context.headers[ key ] = cgi[ cgiKey ]; + } + } + + return context; + } + + /** + * Gets server context including Lucee and Java versions. + */ + private struct function _getServerContext() { + var contexts = { + "runtime" = { + "name" = "Lucee", + "version" = server.lucee.version ?: "unknown" + }, + "os" = { + "name" = server.os.name ?: "unknown", + "version" = server.os.version ?: "unknown" + } + }; + + // Add Java version + if ( structKeyExists( server, "java" ) && structKeyExists( server.java, "version" ) ) { + contexts[ "runtime" ][ "java_version" ] = server.java.version; + } + + return contexts; + } + + /** + * Parses CFML exception stack trace into Sentry format. + */ + private struct function _parseStackTrace( required any exception ) { + var frames = []; + + if ( structKeyExists( arguments.exception, "tagContext" ) && isArray( arguments.exception.tagContext ) ) { + for ( var frame in arguments.exception.tagContext ) { + arrayAppend( frames, { + "filename" = frame.template ?: "", + "lineno" = frame.line ?: 0, + "function" = frame.id ?: "", + "in_app" = true + } ); + } + } + + return { + "frames" = frames + }; + } + + /** + * Normalizes level names to match Sentry's expected format. + * Handles common variations like "warn" -> "warning", "ERROR" -> "error", etc. + */ + private string function _normalizeSentryLevel( required string level ) { + var normalized = lCase( trim( arguments.level ) ); + + // Map common variations to Sentry levels + switch ( normalized ) { + case "warn": + return "warning"; + case "fatal": + case "critical": + return "fatal"; + case "err": + return "error"; + default: + // Return as-is for: debug, info, warning, error, fatal + return normalized; + } + } + +} diff --git a/apps/updateserver/services/VersionUtils.cfc b/apps/updateserver/services/VersionUtils.cfc index cecb088..06b3597 100644 --- a/apps/updateserver/services/VersionUtils.cfc +++ b/apps/updateserver/services/VersionUtils.cfc @@ -1,5 +1,8 @@ component { + static { + static.DEBUG = (server.system.environment.DEBUG ?: false); + } /** * Takes an OSGi compatible version string and returns as a "sortable" @@ -119,60 +122,71 @@ component { }; } + + public static function versionArrayToStruct(array versions) { + var rtn=[:]; + loop array=arguments.versions item="local.data" { + rtn[data.version] = data; + } + return rtn; + } + public static function matchVersion( versions, type, version, distribution ){ - //systemOutput(versions.toJson(), true); - //systemOutput("", true); - //systemOutput("-------[#type#][#version#][#distribution#]------------", true); - var arrVersions = structKeyArray( arguments.versions ).reverse(); + if(isArray(versions)) { + // convert array to struct + arguments.versions = versionArrayToStruct(arguments.versions); + } + var arrVersions = structKeyArray( arguments.versions ).reverse(); + loop array="#arrVersions#" index="local.i" value="local.v" { var _version = versions[ local.v ].version; var _type = listToArray( _version, "-" ); // [ "6.2.0.317", "RC" ] - //systemOutput({_version, _type}, true); + //if(static.DEBUG) systemOutput({_version, _type}, true); if ( arrayLen( _type ) eq 2 && arguments.type eq "stable" ){ // version has a suffix, i.e. 6.2.1.55-SNAPSHOT, stable versions have no suffix - //systemOutput("version has a suffix, not stable", true); + //if(static.DEBUG) systemOutput("version has a suffix, not stable", true); continue; } else if ( len( arguments.type ) gt 0 && arguments.type neq "stable" ){ if ( arrayLen( _type ) eq 1 || ( _type[ 2 ] neq arguments.type) ){ - //systemOutput("wrong type", true); + //if(static.DEBUG) systemOutput("wrong type", true); continue; } } if ( len( arguments.version ) eq 0 && structKeyExists( versions[ local.v ], arguments.distribution ) ){ - //systemOutput("match for the first version for the requested distribution", true); + //if(static.DEBUG) systemOutput("match for the first version for the requested distribution", true); return v; } var versionMatches = findNoCase( arguments.version, _version ); if ( versionMatches neq 1 ) { - //systemOutput("requested version prefix does not match this version", true); + //if(static.DEBUG) systemOutput("requested version prefix does not match this version", true); continue; } // at this point, the versions prefix match if ( versionMatches eq 1 && len( _type[ 1 ] ) eq len( arguments.version ) ) { - //systemOutput("exact version match", true); + //if(static.DEBUG) systemOutput("exact version match", true); if ( structKeyExists( versions[ local.v ], arguments.distribution ) ){ - //systemOutput("exact version match", true); + //if(static.DEBUG) systemOutput("exact version match", true); return v; } else { - //systemOutput("exact version match, no distribution", true); + //if(static.DEBUG) systemOutput("exact version match, no distribution", true); return ""; } } else { // avoid 6.2.1.55 matching 6.2.1.5 if ( ( len( arguments.version ) lt len( _type[ 1 ] ) && mid( _type[ 1 ], len( arguments.version ) + 1, 1 ) neq "." )) { - //systemOutput("avoid partial match", true); + //if(static.DEBUG) systemOutput("avoid partial match", true); continue; } } if ( structKeyExists( versions[ local.v ], arguments.distribution ) ){ return v; // match on version and distribution } else { - //systemOutput("match but missing distribution", true); + //if(static.DEBUG) systemOutput("match but missing distribution", true); continue; // } } @@ -197,4 +211,93 @@ component { return (v[4] <= arguments.build); } + + public boolean static function isVersion(required string version) { + try{ + toVersion(version); + return true; + } + catch(e) { + return false; + } + } + + public struct static function toVersion(required string version, boolean ignoreInvalidVersion=false){ + local.arr=listToArray(arguments.version,'.'); + if(arr.len()==3) { + arr[4]="0"; + } + if(arr.len()!=4 || !isNumeric(arr[1]) || !isNumeric(arr[2]) || !isNumeric(arr[3])) { + if(ignoreInvalidVersion) return {}; + throw ("version number ["&arguments.version&"] is invalid"); + } + local.sct={major:arr[1]+0,minor:arr[2]+0,micro:arr[3]+0,qualifier_appendix:"",qualifier_appendix_nbr:100}; + + // qualifier has an appendix? (BETA,SNAPSHOT) + local.qArr=listToArray(arr[4],'-'); + if(qArr.len()==1 && isNumeric(qArr[1])) local.sct.qualifier=qArr[1]+0; + else if(qArr.len()==2 && isNumeric(qArr[1])) { + sct.qualifier=qArr[1]+0; + sct.qualifier_appendix=qArr[2]; + if(sct.qualifier_appendix=="SNAPSHOT")sct.qualifier_appendix_nbr=0; + else if(sct.qualifier_appendix=="BETA")sct.qualifier_appendix_nbr=50; + else sct.qualifier_appendix_nbr=75; // every other appendix is better than SNAPSHOT + } + else throw ("version number ["&arguments.version&"] is invalid"); + sct.pure= + sct.major + &"."&sct.minor + &"."&sct.micro + &"."&sct.qualifier; + sct.display= + sct.pure + &(sct.qualifier_appendix==""?"":"-"&sct.qualifier_appendix); + + sct.sortable=repeatString("0",2-len(sct.major))&sct.major + &"."&repeatString("0",3-len(sct.minor))&sct.minor + &"."&repeatString("0",3-len(sct.micro))&sct.micro + &"."&repeatString("0",4-len(sct.qualifier))&sct.qualifier + &"."&repeatString("0",3-len(sct.qualifier_appendix_nbr))&sct.qualifier_appendix_nbr; + return sct; + } + + private boolean static function isNewer(required struct left, required struct right ){ + // major + if(left.major>right.major) return true; + if(left.majorright.minor) return true; + if(left.minorright.micro) return true; + if(left.microright.qualifier) return true; + if(left.qualifierright.qualifier_appendix_nbr) return true; + if(left.qualifier_appendix_nbrright.qualifier_appendix) return true; + if(left.qualifier_appendix 0 ) s3.reset(); // update version cache with new artifacts from s3 + } + catch (e){ + logger("----------------------------------------------"); + logger(cfcatch.stacktrace); + writeLog( text=e.message, exception=e, type="error" ); + } + finally { + if( !isNull( localLuceeJar ) && fileExists( localLuceeJar ) ) + fileDelete( localLuceeJar ); + } + } + } catch(e) { + logger( "--- " & arguments.version & " already building, skipping, #e.message#"); + } + } + + /* + MARK: Create LCO + */ + private function createLCO( jar, version ) { + var trg = variables.s3Root & "/org/lucee/lucee/#version#/lucee-#version#.lco"; + if ( fileExists( trg ) ) { + logger("--- " & trg & " already built, skipping" ); + return trg; + } + try { + var temp = getTempDirectory( arguments.version ); + var lco= temp & "lucee-" & version & ".lco"; + + fileCopy( "zip://" & jar & "!core/core.lco", lco ); // now extract + fileMove( lco, trg ); + } + catch( e ){ + logger(text=e.message, type="error", exception=e); + trg = e.message; + } + finally { + if (!isNull(temp) && directoryExists(temp)) directoryDelete(temp,true); + } + return trg; + } + + /* + MARK: Create WAR + */ + private function createWar( jar, version ) { + //logger("--- createWar ---" ); + var war=variables.s3Root & "/org/lucee/lucee/#version#/lucee-#version#.war"; + if ( fileExists( war ) ) { + logger("--- " & war & " already built, skipping" ); + return war; + } + else { + logger("--- " & war & " not found, creating"); + } + var temp = getTempDirectory( arguments.version ); + var warTmp=temp & "lucee-" & version & "-temp-" & createUniqueId() & ".war"; + var curr=getDirectoryFromPath( getCurrentTemplatePath() ); + var warTemplateFolder = getWarTemplate( arguments.version ); + + try { + // temp directory + // create paths and dir if necessary + var build={}; + loop list="extensions,common,website,war" item="local.name" { + var tmp=curr & "build/" & name & "/"; + if ( !directoryExists( tmp ) ){ + if ( name == "extensions" ){ + directoryCreate( tmp, true ); + } else { + throw( message="Required build directory missing: " & tmp ); + } + } + if ( name == "war" ){ + tmp = curr & "build/" & warTemplateFolder & "/"; + } + build[ name ] = tmp; + + } + //logger( "---- createWar", t); + //logger( build, t); + + // let's zip it + zip action="zip" file=warTmp overwrite=true { + zipparam source=build["extensions"] filter="*.lex" prefix="WEB-INF/lucee-server/context/deploy"; + zipparam source=jar entrypath="WEB-INF/lib/lucee.jar"; + zipparam source=build["common"]; + zipparam source=build["website"]; + zipparam source=build["war"]; + } + fileMove (warTmp, war ); + } + catch( e ){ + logger(text=e.message, type="error", exception=e); + trg = e.message; + } + finally { + if (!isNull(temp) && directoryExists(temp)) directoryDelete(temp,true); + } + return war; + } + + /* + MARK: Create LIGHT + @param boolean noArchives aka zero + */ + + private function createLight(jar, version, boolean toS3=true, tempDir, boolean noArchives=false) { + var sep=server.separator.file; + var suffix = arguments.noArchives ? "zero" : "light"; + + var trg=variables.s3Root & "/org/lucee/lucee/#version#/lucee-#version#-#suffix#.jar"; + if ( fileExists( trg ) ) { + // avoid double handling for forgebox light builds + logger("--- " & trg & " already built, skipping" ); + if ( len( arguments.tempDir ) ) { + var tempLight = getTempFile( arguments.tempDir, "lucee-#suffix#-" & version, "jar"); + fileCopy( trg, tempLight); // create a local temp file from s3 + return tempLight; + } else { + return trg; + } + } + var temp = getTempDirectory( arguments.version ); + var s = getTickCount(); + try { + var tmpLoader=temp & "lucee-loader-" & createUniqueId(); // the jar + directoryCreate( tmpLoader ); + + // unzip + try{ + zip action="unzip" file=jar destination=tmpLoader; + } + catch(e) { + fileDelete(jar); + return ""; + } + // rewrite trg + var extDir=tmpLoader & sep & "extensions"; + if ( directoryExists( extDir ) ) directoryDelete(extDir,true); // deletes directory with all files inside + directoryCreate( extDir ); // create empty dir again (maybe Lucee expect this directory to exist) + + // unzip core + var lcoFile=tmpLoader & sep & "core" & sep & "core.lco"; + var tmpCore=temp & "lucee-core-" & createUniqueId(); // the jar + directoryCreate(tmpCore); + zip action="unzip" file=lcoFile destination=tmpCore; + + if ( arguments.noArchives ) { + // delete the lucee-admin.lar and lucee-docs.lar, i.e. lucee zero + var lightContext = tmpCore & sep & "resource/context" & sep; + loop list="lucee-admin.lar,lucee-doc.lar" item="local.larFile" { + fileDelete( lightContext & larFile ); + } + } + + // rewrite manifest + var manifest=tmpCore & sep & "META-INF" & sep&"MANIFEST.MF"; + var content=fileRead(manifest); + var index=find('Require-Extension',content); + if(index>0) content=mid(content,1,index-1)&variables.NL; + fileWrite(manifest,content); + + // zip core + if ( fileExists( lcoFile ) ) fileDelete( lcoFile ); + zip action="zip" source=tmpCore file=lcoFile; + // zip loader + var tmpLoaderFile=temp&"lucee-loader-"&createUniqueId()&".jar"; + zip action="zip" source=tmpLoader file=tmpLoaderFile; + + //if(fileExists(light)) fileDelete(light); + if ( arguments.toS3 ) { + fileMove( tmpLoaderFile, trg ); + } else if ( len( arguments.tempDir ) ) { + // forgebox light build needs a local copy, finally delete that working directory + var tempLight = getTempFile( arguments.tempDir, "lucee-#suffix#-" & version, "jar"); + fileMove( tmpLoaderFile, tempLight); + tmpLoaderFile = tempLight; + } + } + catch( e ){ + logger(text=e.message, type="error", exception=e); + trg = e.message; + } + finally { + if (!isNull(temp) && directoryExists(temp)) directoryDelete(temp,true); + } + return arguments.toS3 ? trg : tmpLoaderFile; + } + + /* + MARK: Create EXPRESS + */ + private string function createExpress(required jar,required string version) { + var sep=server.separator.file; + var trg = variables.s3Root & "/org/lucee/lucee/#version#/lucee-#version#-express.zip"; + if ( fileExists( trg ) ) { + logger("--- " & trg & " already built, skipping" ); + return trg; + } + var temp = getTempDirectory( arguments.version ); + //todo this can overlapp? + var curr=getDirectoryFromPath(getCurrentTemplatePath()); + + // website trg + var zipTmp=temp & "lucee-express-" & version & "-temp-" &createUniqueId() & ".zip"; + var tmpTom="#temp#tomcat"; + // Create the express zip + try { + // extension directory + var extDir = curr & ("build/extensions/"); + if (!directoryExists(extDir)) directoryCreate(extDir); + + // common directory + var commonDir = curr & ("build/common/"); + //if (!directoryExists(commonDir)) directoryCreate(commonDir); + + // website directory + var webDir = curr & ("build/website/"); + //if (!directoryExists(webDir)) directoryCreate(webDir); + + var expressTemplates = getExpressTemplates(); // at this point it should be already cached in the application scope + // unpack the lucee tomcat template + var local_tomcat_templates = curr & "build/servers" + if ( checkVersionGTE( arguments.version, 6, 2, 1 ) ) { + logger("Using Tomcat 11" ); + if ( !structKeyExists( expressTemplates, 'tomcat-11' ) ) + throw( message="No Tomcat 11 express template found for version #arguments.version#" ); + zip action="unzip" file="#local_tomcat_templates#/#expressTemplates['tomcat-11']#" destination=tmpTom; + } else if ( checkVersionGTE( arguments.version, 6, 2 ) ) { + logger("Using Tomcat 10" ); + if ( !structKeyExists( expressTemplates, 'tomcat-10' ) ) + throw( message="No Tomcat 10 express template found for version #arguments.version#" ); + zip action="unzip" file="#local_tomcat_templates#/#expressTemplates['tomcat-10']#" destination=tmpTom; + } else { + logger("Using Tomcat 9" ); + if ( !structKeyExists( expressTemplates, 'tomcat-9' ) ) + throw( message="No Tomcat 9 express template found for version #arguments.version#" ); + zip action="unzip" file="#local_tomcat_templates#/#expressTemplates['tomcat-9']#" destination=tmpTom; + } + + // let's zip it + zip action="zip" file=zipTmp overwrite=true { + // tomcat server + zipparam source=temp&"tomcat"; + // extensions to bundle + zipparam source=extDir filter="*.lex" prefix="lucee-server/context/deploy"; + // jars + zipparam source=jar entrypath="lib/ext/#listLast(jar, "/\")#"; + // common files + zipparam source=commonDir; + // website files + zipparam source=webDir prefix="webapps/ROOT"; + } + fileMove( zipTmp , trg ); + } + catch( e ){ + logger(text=e.message, type="error", exception=e); + trg = e.message; + } + finally { + if (!isNull(temp) && directoryExists(temp)) directoryDelete(temp,true); + } + return trg; + } + + /* + MARK: Create FORGEBOX + */ + private string function createForgeBox(required jar,required string version, boolean light=false) { + var trg = variables.s3Root & "/org/lucee/lucee/#version#/lucee-#version#-forgebox#( light ? '-light' : '' )#.zip"; + if ( fileExists( trg ) ) { + logger("--- " & trg & " already built, skipping" ); + return trg; + } + var sep = server.separator.file; + var temp = getTempDirectory( arguments.version ); + var curr = getDirectoryFromPath(getCurrentTemplatePath()); + + var zipTmp=temp & "forgebox#( light ? '-light' : '' )#-" & version & "-temp-" & createUniqueId() & ".zip"; + try { + // extension directory + var extDir=curr & "/build/extensions/"; + if(!directoryExists(extDir)) directoryCreate(extDir); + + // common directory + var commonDir=curr & "/build/common/"; + //if(!directoryExists(commonDir)) directoryCreate(commonDir); + + // war directory + var warDir = curr & "/build/" & getWarTemplate( arguments.version ) & "/"; + + // create the war + var war=temp & "/engine.war"; + if ( arguments.light ) { + var lightJar = createLight(jar, version, false, temp); + if ( !fileExists( lightJar ) ) + throw "ERROR: forgebox light, createLight didn't produce a jar [#lightJar#]"; + } + + zip action="zip" file=war overwrite=true { + zipparam source=extDir filter="*.lex" prefix="WEB-INF/lucee-server/context/deploy"; + zipparam source=( light ? lightJar: jar) entrypath="WEB-INF/lib/lucee#( arguments.light ? '-light' : '' )#.jar"; + zipparam source=commonDir; + zipparam source=warDir; + } + + // create the json + // Turn 1.2.3.4 into 1.2.3+4 and 1.2.3.4-rc into 1.2.3-rc+4 + var v=reReplace( arguments.version, '([0-9]*\.[0-9]*\.[0-9]*)(\.)([0-9]*)(-.*)?', '\1\4+\3' ); + var json = temp & "/box.json"; + var boxJson = [ + "name":"Lucee #( light ? 'Light' : '' )# CF Engine", + "version":"#v#", + "createPackageDirectory":false, + //"location":"https://cdn.lucee.org/rest/update/provider/forgebox/#arguments.version##( light ? '?light=true' : '' )#", + "slug":"lucee#( light ? '-light' : '' )#", + "shortDescription":"Lucee #( light ? 'Light' : '' )# WAR engine for CommandBox servers.", + "type":"cf-engines" + ]; + if ( checkVersionGTE( arguments.version, 7 ) ){ + //logger( "Using JakartaEE" ); + boxJson[ "JakartaEE" ] = true; + } + fileWrite( json, boxJson.toJson() ); + //logger( boxJson.toJson(), t); + + // create the war + zip action="zip" file=zipTmp overwrite=true { + zipparam source=war; + zipparam source=json; + } + fileMove( zipTmp, trg ); + } + catch( e ){ + logger(text=e.message, type="error", exception=e); + trg = e.message; + } + finally { + if (!isNull(temp) && directoryExists(temp)) directoryDelete(temp,true); + } + return trg; + } + + /* + MARK: Helpers + */ + + private function checkVersionGTE( version, major, minor, patch="" ){ + var v = listToArray( arguments.version, "." ); + if ( v[ 1 ] gt arguments.major ) { + return true; + } else if ( v[ 1 ] eq arguments.major && v[ 2 ] gte arguments.minor ) { + if ( len( arguments.patch ) ) + return v[ 3 ] gte arguments.patch; + else + return true; + } + return false; + } + + private function getWarTemplate( version ){ + if ( checkVersionGTE( arguments.version, 7 ) ) + return "war-7.0"; // jakarta, no lucee servlet + else if ( checkVersionGTE( arguments.version, 6, 2 ) ) + return "war-6.2"; // javax & jakarta, no lucee servlet + else + return "war"; // javax + } + + private function logger( string text, any exception, type="info", boolean forceSentry=false ){ + // var log = arguments.text & chr(13) & chr(10) & callstackGet('string'); + if ( !isNull(arguments.exception ) ){ + if (static.DEBUG) { + if ( len(arguments.text ) ) systemOutput( arguments.text, true ); + systemOutput( arguments.exception, true ); + } else { + writeLog( text=arguments.text, type=arguments.type, log="exception", exception=arguments.exception ); + // Send errors and warnings to Sentry (case insensitive check) + var normalizedType = lCase( arguments.type ); + if ( normalizedType == "error" || normalizedType == "warning" || normalizedType == "warn" ) { + try { + var sentryExtra = {}; + // Include custom text as context if provided + if ( len( arguments.text ) ) { + sentryExtra[ "logText" ] = arguments.text; + } + application.sentryLogger.logException( + exception = arguments.exception, + level = arguments.type, + extra = sentryExtra + ); + } catch ( any e ) { + // Don't let Sentry failures break anything + } + } + } + } else { + if (static.DEBUG) { + systemOutput( arguments.text, true); + } else { + writeLog( text=arguments.text, type=arguments.type, log=variables.providerLog ); + // Send to Sentry if forceSentry is true + if ( arguments.forceSentry ) { + try { + application.sentryLogger.logMessage( message=arguments.text, level=arguments.type ); + } catch ( any e ) { + // Don't let Sentry failures break anything + } + } + } + } + } + + public function getExpressTemplates(){ + if ( !structKeyExists( application, "expressTemplates" ) ) { + application.expressTemplates = new expressTemplates().getExpressTemplates( s3Root ); + } + return application.expressTemplates; + } + +} \ No newline at end of file diff --git a/apps/updateserver/services/legacy/MavenRepo.cfc b/apps/updateserver/services/legacy/MavenRepo.cfc index 866dd01..bb0341c 100644 --- a/apps/updateserver/services/legacy/MavenRepo.cfc +++ b/apps/updateserver/services/legacy/MavenRepo.cfc @@ -1,4 +1,8 @@ component { + + static { + static.DEBUG = (server.system.environment.DEBUG ?: false); + } variables.listPattern= "https://oss.sonatype.org/service/local/lucene/search"; //defaultRepo="https://oss.sonatype.org/content/repositories/releases"; variables.defaultRepo="https://repo1.maven.org/maven2"; @@ -32,7 +36,7 @@ component { lock name="fetch-MavenList" timeout=10 type="exclusive" throwOnTimeout=true{ var t= getTickcount(); mavenList = _list(argumentCollection=arguments); - systemOutput("maven.list() took: " & getTickCount()-t & "ms", true); + if(static.DEBUG) systemOutput("maven.list() took: " & getTickCount()-t & "ms", true); application.cacheMavenList = mavenList; application.cacheMavenListUpdated = now(); return application.cacheMavenList; @@ -61,7 +65,7 @@ component { info.url=infoURL; var fi=dir&"info.json"; if(!fileExists(fi) || fileRead(fi)!=info.totalCount) { - systemOutput( "Maven.list: update required!", true); + if(static.DEBUG) systemOutput( "Maven.list: update required!", true); update=true; } @@ -171,7 +175,7 @@ component { local.detail.groupId=g; local.detail.artifactId=a; local.detail.version=v; - systemOutput(id&":"&(getTickCount()-start),true,true); + if(static.DEBUG) systemOutput(id&":"&(getTickCount()-start),true,true); } arrayAppend(arr,detail); @@ -341,7 +345,7 @@ component { local.jar=getArtifactDirectory()&"lucee-light-"&version&".jar"; // the jar if(!structKeyExists(url,"makefresh") && fileExists(jar)) return jar; - setting requesttimeout="10000"; + setting requesttimeout="1000"; var loader=getLoader(version); //try{ createLight(loader,jar); @@ -698,7 +702,7 @@ component { boxJson[ "JakartaEE" ] = true; } fileWrite( json, boxJson.toJson() ); - //systemOutput( boxJson.toJson(), true ); + //if(static.DEBUG) systemOutput( boxJson.toJson(), true ); // create the war zip action="zip" file=zipTmp overwrite=true { @@ -724,7 +728,7 @@ component { arrayAppend(checks, sct.version); if(sct.version==arguments.version) { if(extended) sct.sources=getSources(sct.repository,sct.version); - // systemOutput("maven.get() took " & numberFormat(getTickCount()-s) & "ms", true); + // if(static.DEBUG) systemOutput("maven.get() took " & numberFormat(getTickCount()-s) & "ms", true); return sct; } } diff --git a/apps/updateserver/services/legacy/S3.cfc b/apps/updateserver/services/legacy/S3.cfc index 6cf2bbf..a1b00ed 100644 --- a/apps/updateserver/services/legacy/S3.cfc +++ b/apps/updateserver/services/legacy/S3.cfc @@ -1,30 +1,31 @@ component { + static { + static.DEBUG = (server.system.environment.DEBUG ?: false); + } + variables.providerLog = "update-provider"; variables.NL=" "; public function init(s3Root) { variables.s3Root=arguments.s3Root; + if ( !structKeyExists( application, "s3VersionDetail" )) { + application.s3VersionDetail = {}; + } + if ( !structKeyExists( application, "s3Versions" )) { + application.s3Versions = {}; + } } public void function reset() { - systemOutput( "s3.reset()", true ); - structDelete( application, "s3VersionData", false ); + logger( "s3.reset()"); + structDelete( application, "s3Versions", false ); + structDelete( application, "s3VersionDetail", false ); structDelete( application, "expressTemplates", false ); - } - - private function logger( string text, any exception, type="info" ){ - var log = arguments.text & chr(13) & chr(10) & callstackGet('string'); - if ( !isNull(arguments.exception ) ){ - WriteLog( text=arguments.text, type=arguments.type, log=variables.providerLog, exception=arguments.exception ); - systemOutput( arguments.exception, true, true ); - } else { - WriteLog( text=arguments.text, type=arguments.type, log=variables.providerLog ); - systemOutput( arguments.text, true, true ); - } + getVersions(flush=true); } /* - MARK: Get Versions + MARK: GetVersions */ public function getVersions(boolean flush=false) { @@ -35,169 +36,94 @@ component { lock name="check-version-cache" timeout="2" throwOnTimeout="false" { if (!directoryExists(cacheDir)) directoryCreate(cacheDir); - if ( isNull(application.s3VersionData) && fileExists( cacheDir & cacheFile ) ){ - systemOutput("s3List.versions load from cache", true); - application.s3VersionData = - sortVersions(deserializeJSON( fileRead(cacheDir & cacheFile), false )); + if ( isNull(application.s3Versions) && fileExists( cacheDir & cacheFile ) ){ + systemOutput("s3List.versions load from cache"); + application.s3Versions = deserializeJSON( fileRead(cacheDir & cacheFile), false ); } } - if(!flush && !isNull(application.s3VersionData)) - return application.s3VersionData; + if ( !flush && !isEmpty(application.s3Versions) ) + return application.s3Versions; lock name="read-version-metadata" timeout="2" throwOnTimeout="false" { setting requesttimeout="1000"; var runid = createUniqueID(); var start = getTickCount(); - - systemOutput("s3Versions.list [#runId#] START #numberFormat(getTickCount()-start)#ms",1,1); - - // systemOutput(variables.s3Root,1,1); try { - var qry=directoryList(path:variables.s3Root,listInfo:"query",filter:function (path){ - var ext=listLast(path,'.'); - var name=listLast(path,'\/'); - - if(ext=='lco') return true; - if(ext=='war' && left(name,6)=='lucee-') return true; - if(ext=='exe' && left(name,6)=='lucee-') return true; // lucee-4.5.3.020-pl0-windows-installer.exe - if(ext=='run' && left(name,6)=='lucee-') return true; // lucee-4.5.3.020-pl0-windows-installer.exe - if(ext=='jar' && left(name,6)=='lucee-') return true; - if(ext=='zip' && (left(name,6)=='lucee-' || left(name,9)=='forgebox-')) return true; - /* - if(ext=='jar' && left(name,6)=='lucee-' || left(name,12)!='lucee-light-') { - return true; - }*/ - return false; - }); - } catch (e){ - systemOutput("error directory listing versions on s3", true); - systemOutput(e, true); - if(isNull(application.s3VersionData)) - return application.s3VersionData; - throw "cannot read versions from s3 directory"; - } - systemOutput("s3Versions.list [#runId#] FETCHED #numberFormat(getTickCount()-start)#ms, #qry.recordcount# files on s3 found",1,1); - //dump(qry); - var data=structNew("linked"); - // first we get all - var patterns=structNew('linked'); - patterns['express']='lucee-express-'; - patterns['light']='lucee-light-'; - patterns['zero']='lucee-zero-'; - patterns['fbl']='forgebox-light-'; - patterns['fb']='forgebox-'; - patterns['jars']='lucee-jars-'; - patterns['jar']='lucee-'; - - loop query=qry { - var ext=listLast(qry.name,'.'); - var version=""; - // core (.lco) - var name=qry.name; - if (ext=='lco') { - var version=mid(qry.name,1,len(qry.name)-4); - var type="lco"; - } - else if (ext=='exe') { - var _installer = services.VersionUtils::parseInstallerFilename( qry.name ); - var version = _installer.version; - var type = _installer.type; - } - else if (ext=='run') { - var _installer = services.VersionUtils::parseInstallerFilename( qry.name ); - var version = _installer.version; - var type = _installer.type; - } - else if(ext=='war') { - var version=mid(qry.name,7,len(qry.name)-10); - var type="war"; - } - // all others - else { - loop struct=patterns index="local.t" item="local.prefix" { - var l=len(prefix); - if(left(qry.name,l)==prefix) { - var version=mid(qry.name,l+1,len(qry.name)-4-l); - var type=t; - if(type=="jars") type="jar"; - break; + systemOutput("s3Versions.list [#runId#] START #numberFormat(getTickCount()-start)#ms",1,1); + var data = getLuceeVersionsListS3(); + if ( len( data ) gt 0 ){ // only cache good data + fileWrite( cacheDir & cacheFile, serializeJSON( data, false) ); + if ( serializeJson( application.s3Versions ) neq SerializeJson( data ) ){ + // only ping on change + application.s3Versions = data; + if ( structKeyExists( application, "downloadsUrl" ) ){ + var downloadRefeshUrl = "#application.downloadsUrl#?type=snapshots&reset=force"; + logger("Versions updated, pinging [#downloadRefeshUrl#]"); + try { + cfhttp( url=downloadRefeshUrl ); + } catch (e){ + logger(message="error returned updating download server [#downloadRefeshUrl#]", exception=e, type="error"); + } + } else { + logger("Versions updated, not pinging due to missing application.downloadsUrl"); } } } - - // check version - var arrVersion=listToArray(version,'.'); - if( arrayLen(arrVersion)!=4 || - !isNumeric(arrVersion[1]) || - !isNumeric(arrVersion[2]) || - !isNumeric(arrVersion[3])) continue; - - // hide 7.0.0.202 stable - if (arrVersion[1] == 7 - && arrVersion[2] == 0 - && arrVersion[3] == 0 - && arrVersion[4] == 202) continue; - - var arrPatch=listToArray(arrVersion[4],'-'); - if( arrayLen(arrPatch)>2 || - arrayLen(arrPatch)==0 || - !isNumeric(arrPatch[1])) continue; - - if(arrayLen(arrPatch)==2 && - arrPatch[2]!="SNAPSHOT" && - arrPatch[2]!="BETA" && - arrPatch[2]!="RC" && - arrPatch[2]!="ALPHA") continue; - - var vs=services.VersionUtils::toVersionSortable(version); - //if(isNull(data[version])) data[version]={}; - data[vs]['version']=version; - data[vs][type]=name; - //data[version]['date-'&type]=qry.dateLastModified; - //data[version]['size-'&type]=qry.size; + application.s3Versions = data; + } catch (e){ + systemOutput( "error directory listing versions on s3", true ); + throw( message="cannot read versions from s3 directory", cause=e ); } - systemOutput("s3Versions.list [#runId#] SORT #numberFormat(getTickCount()-start)#ms, #len(data)# versions found ",1,1); - // sort - var _data = sortVersions(data); - if ( len(_data) gt 0 ) // only cache good data - fileWrite(cacheDir & cacheFile, serializeJSON(_data, false) ); - systemOutput("s3Versions.list [#runId#] END #numberFormat(getTickCount()-start)#ms, #len(_data)# versions found",1,1); - - if ( structCount(_data) eq 0 && !isNull(application.s3VersionData) ) - return application.s3VersionData; // emergency hotfix - return application.s3VersionData=_data; + systemOutput("s3Versions.list [#runId#] FETCHED #numberFormat(getTickCount()-start)#ms, #len(data)# files on s3 found",1,1); } - if ( !structKeyExists( application, "s3VersionData" ) ){ + if ( !structKeyExists( application, "s3Versions" ) ){ // lock timed out, still use cache if found if ( fileExists( cacheDir & cacheFile ) ){ systemOutput("s3List.versions load from cache (after lock)", true); - var _data = deserializeJSON( fileRead(cacheDir & cacheFile), false ); - application.s3VersionData = sortVersions(_data); + var data = deserializeJSON( fileRead(cacheDir & cacheFile), false ); + application.s3Versions = data; } else { throw "lock timeout readVersions() no cached found"; } } - return application.s3VersionData; + return application.s3Versions; } - private function sortVersions(data){ - var keys=structKeyArray(data); - arraySort(keys,"textnocase"); - var _data=structNew("linked"); - loop array=keys item="local.k" { - if ( structKeyExists( data[ k ], "version" ) && !isEmpty( data[k][ 'version' ] ) ) - _data[k] = data[k]; + // this simply gets one version from the versions list + public function getLuceeVersionsDetail( version ) { + if ( !structKeyExists( application, "s3VersionDetail" ) ) { + application[ "s3VersionDetail" ] = {}; + } else if ( structKeyExists( application.s3VersionDetail, version ) ) { + return application.s3VersionDetail[ version ]; } - return _data; - } - public function getLatestVersion(boolean flush=false) { - var versions=getVersions(flush); - var keys=structKeyArray(versions); - arraySort(keys,"textnocase"); - return versions[keys[arrayLen(keys)]].version; + var versionInfo = {}; + if ( left( s3Root, 3 ) == "s3:" ) { + versionInfo = luceeVersionsDetailS3( arguments.version ); + } else { + versionInfo = getLocalVersionsDetail( arguments.version ); // this is still faster, as it's reading from the local cache + } + return application.s3VersionDetail[ version ] = versionInfo; + } + + // fallback handling for local testing with Lucee jar files in the root directory + public function getJarPath( version ){ + var jarPath = variables.s3Root & "/org/lucee/lucee/#arguments.version#/lucee-#arguments.version#.jar"; + if ( left( s3Root, 3) == "s3:") { + return jarPath; + } + if (!fileExists(jarPath)){ + var dir = getDirectoryFromPath(jarPath); + if (!directoryExists(dir)) + directoryCreate(dir); + var tmpJar = getLocalVersionsDetail(version).jar; + logger ("local_S3: copying jar [#tmpJar#] to [#jarPath#]"); + fileCopy( tmpJar, jarPath ); + reset(); + } + return jarPath; } public function getExpressTemplates(){ @@ -207,29 +133,33 @@ component { return application.expressTemplates; } + /* + MARK: Add + */ public function add(required string type, required string version) { - setting requesttimeout="10000000"; - var versions=getVersions(true); - var vs=services.VersionUtils::toVersionSortable(version); + setting requesttimeout="1000"; + + logger("-------- add:#type# --------"); + var data = getLuceeVersionsDetail(version); + //logger(data); var mr=new MavenRepo(); - // move the jar to maven if necessary - if(!structKeyExists(versions,vs) || !structKeyExists(versions[vs],'jar')) { - maven2S3(mr,version,versions); - SystemOutput("add: downloaded jar from maven:"&now(),1,1); - versions = getVersions(true); + // move the jar from maven if necessary + if (isNull(data.jar)) { + maven2S3(mr,version); + logger("add: downloaded jar from maven:"&now()); } + //var vs=services.VersionUtils::toVersionSortable(version); // create the artifact try { - if( type != "jar" ){ - SystemOutput("add: createArtifacts (#type#):"&now(),1,1); - createArtifacts(mr,versions[vs],type,true); - versions = getVersions(true); - SystemOutput("add: after creating artifact (#type#):"&now(),1,1); + if ( type != "jar" ){ + logger("add: createArtifacts (#type#):"&now()); + new ArtifactBuilder(s3Root).createArtifacts(mr,version,type,true); + logger("add: artifact created (#type#):"&now()); } } catch(e){ - SystemOutput(e.stacktrace,1,1); + logger(exception=e, text="add: error creating artifacts for version #version# type #type#", type="error"); } } @@ -237,7 +167,7 @@ component { MARK: Add Missing */ public function addMissing(includingForgeBox=false, skipMaven=false) { - setting requesttimeout="1000000"; + setting requesttimeout="1000"; systemOutput("start:"&now(),1,1); var started = getTickCount(); @@ -262,441 +192,195 @@ component { } getExpressTemplates(); // create the missing artifacts - loop struct=s3List index="local.vs" item="local.el" { - createArtifacts(mr,el,"",includingForgeBox); + var builder = new ArtifactBuilder(s3Root); + loop array=s3List item="local.el" { + builder.createArtifacts(mr,el.version,"",includingForgeBox); } systemOutput("build complete, all artifacts created in #numberFormat(getTickCount()-started)#,s",1,1); getVersions(true); //force reset(); } - private function maven2S3(mr,version,all) { + public function buildLatest(includingForgeBox=false) { + var list=getLuceeVersionsListS3( flush=true ); + var latest=list[len(list)].version; + logger("buildLatest: " & latest); + var mr=new MavenRepo(); + new ArtifactBuilder(s3Root).createArtifacts(mr,latest,"",includingForgeBox); + } + + /* + MARK: Helpers + */ + + private function logger( string text, any exception, type="info", boolean forceSentry=false ){ + //var log = arguments.text & chr(13) & chr(10) & callstackGet('string'); + if ( !isNull(arguments.exception ) ){ + + if (static.DEBUG) { + if (len(arguments.text)) systemOutput( arguments.text, true, true ); + systemOutput( arguments.exception, true, true ); + } else { + writeLog( text=arguments.text, type=arguments.type, log="exception", exception=arguments.exception ); + // Send errors and warnings to Sentry (case insensitive check) + var normalizedType = lCase( arguments.type ); + if ( normalizedType == "error" || normalizedType == "warning" || normalizedType == "warn" ) { + try { + var sentryExtra = {}; + // Include custom text as context if provided + if ( len( arguments.text ) ) { + sentryExtra[ "logText" ] = arguments.text; + } + application.sentryLogger.logException( + exception = arguments.exception, + level = arguments.type, + extra = sentryExtra + ); + } catch ( any e ) { + // Don't let Sentry failures break anything + } + } + } + } else { + if (static.DEBUG) { + systemOutput( arguments.text, true, true ); + } else { + writeLog( text=arguments.text, type=arguments.type, log="application" ); + // Send to Sentry if forceSentry is true + if ( arguments.forceSentry ) { + try { + application.sentryLogger.logMessage( message=arguments.text, level=arguments.type ); + } catch ( any e ) { + // Don't let Sentry failures break anything + } + } + } + } + } + + + private function maven2S3(mr,version) { if(left(version,1)<5) return; - // ignore this versions - if(listFind("5.0.0.20-SNAPSHOT,5.0.0.255-SNAPSHOT,5.0.0.256-SNAPSHOT,5.0.0.258-SNAPSHOT,5.0.0.259-SNAPSHOT",version)) { - structDelete(all,version,false); + // ignore these versions + if(listFind("5.0.0.20-SNAPSHOT,5.0.0.255-SNAPSHOT,5.0.0.256-SNAPSHOT,5.0.0.258-SNAPSHOT,5.0.0.259-SNAPSHOT,7.0.0.202",version)) { + logger( "maven2S3 skipping bad version: #version#" ); return; } - var trg=variables.s3Root&"lucee-"&version&".jar"; - + var trg = variables.s3Root & "/org/lucee/lucee/#version#/lucee-#version#.jar"; lock name="download from maven-#version#" timeout="1" { - systemOutput("downloading from maven-#version#",1,1); + logger( "downloading from maven-#version#" ); // add the jar var info=mr.get(version, true); if(isNull(info.sources.jar.src)) { - systemOutput("404:"&version,1,1); - structDelete(all,version,false); + logger("404:"&version); return; } var src=info.sources.jar.src; var date=parseDateTime(info.sources.jar.date); if (!fileExists(src)) { - structDelete(all,version,false); - systemOutput("404:"&src,1,1); + logger("404:"&src); return; } // copy jar from maven to S3 - fileCopy(src,trg); + var dir = getDirectoryFromPath(trg); + if (!directoryExists(dir)) + directoryCreate(dir); + if (!fileExists(trg)) + fileCopy(src,trg); - systemOutput("200:"&trg,1,1); - all[version]['jar']=true; - all[version]['date-jar']=date; + logger("200:"&trg); } } - /* - MARK: Create Artifacts + /** + * checks if file exists on S3 and if so redirect to it, if not it copies it to S3 and the next one will have it there. + * So nobody has to wait that it is copied over */ - private function createArtifacts(mr,s3,specType="",includingForgeBox=true) { - if(left(s3.version,1)<5) return; - - var jarRem=variables.s3Root&"lucee-"&s3.version&".jar"; + private function fromS3(path,name,async=true) { + // if exist we redirect to it + if(!isNull(url.show)) + throw ((!isNull(application.exists[name]) && application.exists[name])&":"&fileExists(variables.s3Root&name)&"->"&(variables.s3Root&name)); + + var hasDef=(!isNull(application.exists[name]) && application.exists[name]); + if(hasDef || fileExists(variables.s3Root&name)) { + application.exists[name]=true; + header statuscode="302" statustext="Found"; + header name="Location" value=variables.cdnURL&name; + return true; + } + // if not exist we make ready for the next + else { - try { - lock name="build-lucee-artifacts-#s3.version#" timeout="1" { - try { - // check and if necessary create other artifacts - var list="lco,war,light,express"; - if(includingForgeBox)list&=",fb,fbl"; - - systemOutput("createArtifacts() Starting ( #s3.version# )",1,1); - var c= 0; - - loop list=list item="local.type" { - if ( len( specType ) && specType!=type ) continue; - if ( structKeyExists( s3, type ) ) continue; - c++; - var s = getTickCount(); - // first we need a local copy of the jar - var lcl=getTempDirectory() & "/lucee-"&s3.version&".jar"; - try{ - if(!fileExists(lcl)) fileCopy(jarRem,lcl); - } - catch(e) { - systemOutput(e,1,1); - continue; - } - // extract lco and copy to S3 - if(type=="lco") { - var result=createLCO(lcl,s3.version); - systemOutput("lco: " & result & " took " & numberFormat(getTickCount()-s) & "ms",1,1); - } - else if(type=="fb") { - var result=createForgeBox(lcl,s3.version,false); - systemOutput("forgebox: " & result & " took " & numberFormat(getTickCount()-s) & "ms",1,1); - //abort; - } - else if(type=="fbl") { - var result=createForgeBox(lcl,s3.version,true); - systemOutput("forgebox-light: " & result & " took " & numberFormat(getTickCount()-s) & "ms",1,1); - } - // create war and copy to S3 - else if(type=="war") { - lock name="build-lucee-war" timeout="10" { - var result=createWar(lcl,s3.version); - } - systemOutput("war: " & result & " took " & numberFormat(getTickCount()-s) & "ms",1,1); - } - // create war and copy to S3 - else if(type=="light") { - var result=createLight(lcl,s3.version); - systemOutput("light: " & result & " took " & numberFormat(getTickCount()-s) & "ms",1,1); - } - else if(type=="express") { - lock name="build-lucee-express" timeout="10" { - var result=createExpress(lcl,s3.version); - } - systemOutput("express: " & result & " took " & numberFormat(getTickCount()-s) & "ms",1,1); - } - else { - systemOutput("unsupported: " & type &":"&s3.version,1,1); - c--; - } + if(async && isNull(url.show)) { + thread src=path trg=variables.s3Root&name { + lock timeout=1000 name=src { + if(!fileExists(trg) && fileSize(src)>100000) // we do this because it was created by a thread blocking this thread + _fileCopy(src,trg); } - systemOutput( "--- " & s3.version & " done #c# artifacts built",1,1); - } - catch (e){ - systemOutput("----------------------------------------------",1,1); - systemOutput(cfcatch.stacktrace,1,1); - writeLog( text=e.message, exception=e, type="error" ); - } - finally { - if(!isNull(lcl) && fileExists(lcl)) fileDelete(lcl); } } - } catch(e) { - systemOutput( "--- " & s3.version & " already building, skipping, #e.message#",1,1); - } - } - - /* - MARK: Create LCO - */ - private function createLCO( jar, version ) { - var trg=variables.s3Root & version & ".lco"; - if ( fileExists( trg ) ) { - systemOutput("--- " & trg & " already built, skipping", true); - } - try { - var temp = getTemp( arguments.version ); - var lco= temp & "lucee-" & version & ".lco"; - - fileCopy( "zip://" & jar & "!core/core.lco", lco ); // now extract - fileMove( lco, trg ); - } - catch( e ){ - logger(text=e.message, type="error", exception=e); - } - finally { - if (!isNull(temp) && directoryExists(temp)) directoryDelete(temp,true); - } - return trg; - } - - /* - MARK: Create WAR - */ - private function createWar( jar, version ) { - var war=variables.s3Root & "lucee-" & version & ".war"; - if ( fileExists( war ) ) { - systemOutput("--- " & war & " already built, skipping", true); - } - var temp = getTemp( arguments.version ); - var warTmp=temp & "lucee-" & version & "-temp-" & createUniqueId() & ".war"; - var curr=getDirectoryFromPath( getCurrentTemplatePath() ); - var warTemplateFolder = getWarTemplate( arguments.version ); - - try { - // temp directory - // create paths and dir if necessary - var build={}; - loop list="extensions,common,website,war" item="local.name" { - var tmp=curr & "build/" & name & "/"; - if ( name == "extensions" && !directoryExists( tmp ) ) - directoryCreate( tmp, true ); - if ( name == "war" ){ - tmp = curr & "build/" & warTemplateFolder & "/"; + else { + var src=path; + var trg=variables.s3Root&name; + lock timeout=1000 name=src { + if(!fileExists(trg) && fileSize(src)>100000) {// we do this because it was created by a thread blocking this thread + _fileCopy(src,trg); + if(!isNull(url.show)) + throw ("fileExists: "&fileExists(src)&" + "&fileExists(trg)); + } } - build[ name ] = tmp; - - } - //systemOutput( "---- createWar", true ); - //systemOutput( build, true ); - - // let's zip it - zip action="zip" file=warTmp overwrite=true { - zipparam source=build["extensions"] filter="*.lex" prefix="WEB-INF/lucee-server/context/deploy"; - zipparam source=jar entrypath="WEB-INF/lib/lucee.jar"; - zipparam source=build["common"]; - zipparam source=build["website"]; - zipparam source=build["war"]; } - fileMove (warTmp, war ); - } - catch( e ){ - logger(text=e.message, type="error", exception=e); - } - finally { - if (!isNull(temp) && directoryExists(temp)) directoryDelete(temp,true); } - return war; + return false; } - /* - MARK: Create LIGHT - */ - - private function createLight(jar, version, boolean toS3=true, tempDir) { - var sep=server.separator.file; - var trg=variables.s3Root & "lucee-light-" & version & ".jar"; - if ( fileExists( trg ) ) { - // avoid double handling for forgebox light builds - systemOutput("--- " & trg & " already built, skipping", true); - var tempLight = getTempFile( arguments.tempDir, "lucee-light-" & version, "jar"); - fileCopy( trg, tempLight); // create a local temp file from s3 - return tempLight; + // wrappers to allow using a local dir for testing without s3 access + private function getLuceeVersionsListS3() { + // TODO this currently is cached internally by lucee MAX_AGE = 10000ms + if ( left( s3Root, 3) == "s3:") { + return luceeVersionsListS3(); } - var temp = getTemp( arguments.version ); - var s = getTickCount(); - try { - var tmpLoader=temp & "lucee-loader-" & createUniqueId(); // the jar - directoryCreate( tmpLoader ); - - // unzip - try{ - zip action="unzip" file=jar destination=tmpLoader; - } - catch(e) { - fileDelete(jar); - return ""; - } - // rewrite trg - var extDir=tmpLoader & sep & "extensions"; - if ( directoryExists( extDir ) ) directoryDelete(extDir,true); // deletes directory with all files inside - directoryCreate( extDir ); // create empty dir again (maybe Lucee expect this directory to exist) - - // unzip core - var lcoFile=tmpLoader & sep & "core" & sep & "core.lco"; - local.tmpCore=temp & "lucee-core-" & createUniqueId(); // the jar - directoryCreate(tmpCore); - zip action="unzip" file=lcoFile destination=tmpCore; - // rewrite manifest - var manifest=tmpCore & sep & "META-INF" & sep&"MANIFEST.MF"; - var content=fileRead(manifest); - var index=find('Require-Extension',content); - if(index>0) content=mid(content,1,index-1)&variables.NL; - fileWrite(manifest,content); - - // zip core - if ( fileExists( lcoFile ) ) fileDelete( lcoFile ); - zip action="zip" source=tmpCore file=lcoFile; - // zip loader - local.tmpLoaderFile=temp&"lucee-loader-"&createUniqueId()&".jar"; - zip action="zip" source=tmpLoader file=tmpLoaderFile; - - //if(fileExists(light)) fileDelete(light); - if (toS3) fileMove(tmpLoaderFile,trg); - } - catch( e ){ - logger(text=e.message, type="error", exception=e); - } - finally { - if (!isNull(temp) && directoryExists(temp)) directoryDelete(temp,true); - } - return toS3?trg:tmpLoaderFile; + return getLocalVersionsList(); } - /* - MARK: Create EXPRESS - */ - private string function createExpress(required jar,required string version) { - var sep=server.separator.file; - var trg = variables.s3Root & "lucee-express-" & version & ".zip"; - if ( fileExists( trg ) ) { - systemOutput("--- " & trg & " already built, skipping", true); - return trg; - } - var temp = getTemp( arguments.version ); - //todo this can overlapp? - var curr=getDirectoryFromPath(getCurrentTemplatePath()); - - // website trg - var zipTmp=temp & "lucee-express-" & version & "-temp-" &createUniqueId() & ".zip"; - var tmpTom="#temp#tomcat"; - // Create the express zip - try { - // extension directory - var extDir = curr & ("build/extensions/"); - if (!directoryExists(extDir)) directoryCreate(extDir); - - // common directory - var commonDir = curr & ("build/common/"); - //if (!directoryExists(commonDir)) directoryCreate(commonDir); - - // website directory - var webDir = curr & ("build/website/"); - //if (!directoryExists(webDir)) directoryCreate(webDir); - - var expressTemplates = getExpressTemplates(); // at this point it should be already cached in the application scope - // unpack the lucee tomcat template - var local_tomcat_templates = curr & "build/servers" - if ( checkVersionGTE( arguments.version, 6, 2, 1 ) ) { - systemOutput("Using Tomcat 11", true); - zip action="unzip" file="#local_tomcat_templates#/#expressTemplates['tomcat-11']#" destination=tmpTom; - } else if ( checkVersionGTE( arguments.version, 6, 2 ) ) { - systemOutput("Using Tomcat 10", true); - zip action="unzip" file="#local_tomcat_templates#/#expressTemplates['tomcat-10']#" destination=tmpTom; - } else { - systemOutput("Using Tomcat 9", true); - zip action="unzip" file="#local_tomcat_templates#/#expressTemplates['tomcat-9']#" destination=tmpTom; - } - - // let's zip it - zip action="zip" file=zipTmp overwrite=true { - // tomcat server - zipparam source=temp&"tomcat"; - // extensions to bundle - zipparam source=extDir filter="*.lex" prefix="lucee-server/context/deploy"; - // jars - zipparam source=jar entrypath="lib/ext/#listLast(jar, "/\")#"; - // common files - zipparam source=commonDir; - // website files - zipparam source=webDir prefix="webapps/ROOT"; - } - fileMove( zipTmp , trg ); - } - catch( e ){ - logger(text=e.message, type="error", exception=e); - } - finally { - if (!isNull(temp) && directoryExists(temp)) directoryDelete(temp,true); - } - return trg; - } - - /* - MARK: Create FORGEBOX - */ - private string function createForgeBox(required jar,required string version, boolean light=false) { - var trg=variables.s3Root & "forgebox#( light ? '-light' : '' )#-" &version & ".zip"; - if ( fileExists( trg ) ) { - systemOutput("--- " & trg & " already built, skipping", true); - return trg; - } - var sep = server.separator.file; - var temp = getTemp( arguments.version ); - var curr = getDirectoryFromPath(getCurrentTemplatePath()); - - var zipTmp=temp & "forgebox#( light ? '-light' : '' )#-" & version & "-temp-" & createUniqueId() & ".zip"; - try { - // extension directory - var extDir=curr & "/build/extensions/"; - if(!directoryExists(extDir)) directoryCreate(extDir); - - // common directory - var commonDir=curr & "/build/common/"; - //if(!directoryExists(commonDir)) directoryCreate(commonDir); - - // war directory - var warDir = curr & "/build/" & getWarTemplate( arguments.version ) & "/"; - - // create the war - var war=temp & "/engine.war"; - if ( light ) local.lightJar=createLight(jar, version, false, temp); - - zip action="zip" file=war overwrite=true { - zipparam source=extDir filter="*.lex" prefix="WEB-INF/lucee-server/context/deploy"; - zipparam source=( light ? lightJar: jar) entrypath="WEB-INF/lib/lucee#( light ? '-light' : '' )#.jar"; - zipparam source=commonDir; - zipparam source=warDir; - } - - // create the json - // Turn 1.2.3.4 into 1.2.3+4 and 1.2.3.4-rc into 1.2.3-rc+4 - var v=reReplace( arguments.version, '([0-9]*\.[0-9]*\.[0-9]*)(\.)([0-9]*)(-.*)?', '\1\4+\3' ); - var json = temp & "/box.json"; - var boxJson = [ - "name":"Lucee #( light ? 'Light' : '' )# CF Engine", - "version":"#v#", - "createPackageDirectory":false, - //"location":"https://cdn.lucee.org/rest/update/provider/forgebox/#arguments.version##( light ? '?light=true' : '' )#", - "slug":"lucee#( light ? '-light' : '' )#", - "shortDescription":"Lucee #( light ? 'Light' : '' )# WAR engine for CommandBox servers.", - "type":"cf-engines" - ]; - if ( checkVersionGTE( arguments.version, 7 ) ){ - systemOutput( "Using JakartaEE", true ); - boxJson[ "JakartaEE" ] = true; - } - fileWrite( json, boxJson.toJson() ); - //systemOutput( boxJson.toJson(), true ); - - // create the war - zip action="zip" file=zipTmp overwrite=true { - zipparam source=war; - zipparam source=json; - } - fileMove( zipTmp, trg ); - } - catch( e ){ - logger(text=e.message, type="error", exception=e); - } - finally { - if (!isNull(temp) && directoryExists(temp)) directoryDelete(temp,true); - } - return trg; + // used for testing, uses a local directory instead of s3 + private function getLocalVersionsList(){ + var versions = new S3local().listVersions(s3root); + // systemOutput(serializeJson(var=versions, compact=false), true); + return versions; } - private function getTemp( string version ){ - var temp = getTempDirectory() & "#arguments.version#-" & createUniqueId() & "/"; - if ( directoryExists( temp ) ) - directoryDelete( temp, true ); - directoryCreate( temp ); - return temp; + private function getLocalVersionsDetail( version ){ + var versions = getVersions(); + var _version = arguments.version; + var detail = arrayFilter(versions, function(item){ + return item.version == _version; + }); + //systemOutput(serializeJson(var=detail, compact=false), true); + if ( arrayLen(detail) eq 0 ) + throw "Version [#arguments.version#] not found"; + return detail[1]; } - private function checkVersionGTE( version, major, minor, patch="" ){ - var v = listToArray( arguments.version, "." ); - if ( v[ 1 ] gt arguments.major ) { - return true; - } else if ( v[ 1 ] eq arguments.major && v[ 2 ] gte arguments.minor ) { - if ( len( arguments.patch ) ) - return v[ 3 ] gte arguments.patch; - else - return true; + /* not used + private function sortVersions(data){ + var keys=structKeyArray(data); + arraySort(keys,"textnocase"); + var _data=structNew("linked"); + loop array=keys item="local.k" { + if ( structKeyExists( data[ k ], "version" ) && !isEmpty( data[k][ 'version' ] ) ) + _data[k] = data[k]; } - return false; + return _data; } - private function getWarTemplate( version ){ - if ( checkVersionGTE( arguments.version, 7 ) ) - return "war-7.0"; // jakarta, no lucee servlet - else if ( checkVersionGTE( arguments.version, 6, 2 ) ) - return "war-6.2"; // javax & jakarta, no lucee servlet - else - return "war"; // javax + public function getLatestVersion(boolean flush=false) { + var versions=getVersions(flush); // missing? + var keys=structKeyArray(versions); + arraySort(keys,"textnocase"); + return versions[keys[arrayLen(keys)]].version; } + */ } \ No newline at end of file diff --git a/apps/updateserver/services/legacy/S3Local.cfc b/apps/updateserver/services/legacy/S3Local.cfc new file mode 100644 index 0000000..e70daf0 --- /dev/null +++ b/apps/updateserver/services/legacy/S3Local.cfc @@ -0,0 +1,128 @@ + +component hint="luceeVersionsListS3 but with a local directory, for local development" { + + function listVersions( dir ){ + var qry=directoryList( path=arguments.dir, recurse= true, listInfo="query", filter=function (path){ + var ext=listLast(path,'.'); + var name=listLast(path,'\/'); + + if(ext=='lco') return true; + if(ext=='war' && left(name,6)=='lucee-') return true; + if(ext=='exe' && left(name,6)=='lucee-') return true; // lucee-4.5.3.020-pl0-windows-installer.exe + if(ext=='run' && left(name,6)=='lucee-') return true; // lucee-4.5.3.020-pl0-windows-installer.exe + if(ext=='jar' && left(name,6)=='lucee-') return true; + if(ext=='zip' && (left(name,6)=='lucee-' || left(name,9)=='forgebox-')) return true; + /* + if(ext=='jar' && left(name,6)=='lucee-' || left(name,12)!='lucee-light-') { + return true; + }*/ + // systemOutput(" skipped file:" & path, true); + return false; + }); + + var data=structNew("linked"); + + // maven files have a different format, type is the suffix, was the prefix before + + var patterns=structNew('linked'); + patterns['express'] = { suffix: 'express', ext: "zip" }; + patterns['light'] = { suffix: 'light', ext: "jar" }; + patterns['zero'] = { suffix: 'zero', ext: "jar" }; + patterns['forgebox-light'] = { suffix: 'forgebox-light', ext: "zip" }; + patterns['forgebox'] = { suffix: 'forgebox', ext: "zip" }; + // patterns['jar'] = { suffix: '', ext: "jar" }; + + loop query=qry { + var type = ""; + var ext=listLast(qry.name,'.'); + var version=""; + // core (.lco) + var name=qry.name; + var _name = mid( listDeleteAt( name, listLen( name, "." ), "." ), 7 ); // strip off the file extension [.zip] and [lucee-] prefix + //systemOutput( "b4:" & qry.directory & "/" & name & " [#_name#]", true); + if ( ext=='lco' ) { + var version = _name; + var type="lco"; + } else if ( ext=='exe' ) { + var _installer = services.VersionUtils::parseInstallerFilename( qry.name ); + var version = _installer.version; + var type = _installer.type; + } else if ( ext=='run' ) { + var _installer = services.VersionUtils::parseInstallerFilename( qry.name ); + var version = _installer.version; + var type = _installer.type; + } else if ( ext=='war' ) { + var version = _name ; + var type="war"; + } + // all others + else { + loop struct=patterns key="local.t" value="local.pattern" { + if ( pattern.ext != ext ) continue; + var s = len( pattern.suffix ); + //systemOutput( "----- suffix " & pattern.suffix & " #s# [" & right( _name, s ) & "] " & _name, true); + if (right( _name, s ) == pattern.suffix ) { + var version = mid( _name, 1, len( _name )-s); // ignore the leading - + var type=t; + if(type=="jars") type="jar"; + break; + } + } + // hard to match no suffix + if ( len( type ) eq 0 and ext eq "jar" ) { + version = _name; + type = "jar"; + } + } + + //systemOutput( "mid:" & qry.directory & "/" & name & " - " & version & " - " & type, true); + + // check version + var arrVersion=listToArray(version,'.'); + if ( arrayLen(arrVersion)!=4 || + !isNumeric(arrVersion[1]) || + !isNumeric(arrVersion[2]) || + !isNumeric(arrVersion[3])) continue; + + // hide 7.0.0.202 stable + if (arrVersion[1] == 7 + && arrVersion[2] == 0 + && arrVersion[3] == 0 + && arrVersion[4] == 202) continue; + + var arrPatch=listToArray(arrVersion[4],'-'); + if ( arrayLen(arrPatch)>2 || + arrayLen(arrPatch)==0 || + !isNumeric(arrPatch[1])) continue; + + if (arrayLen(arrPatch)==2 && + arrPatch[2]!="SNAPSHOT" && + arrPatch[2]!="BETA" && + arrPatch[2]!="RC" && + arrPatch[2]!="ALPHA") continue; + + var vs=services.VersionUtils::toVersionSortable(version); + //if(isNull(data[version])) data[version]={}; + if ( !structKeyExists( data, vs ) ) { + data[vs]= {}; + data[vs]['version']=version; + } + data[vs][type]=qry.directory & "/" & name; + if ( type=="jar" ){ + data[vs]['lastModified']=qry.dateLastModified; + data[vs]['size']=qry.size; + } + // systemOutput( "after" & qry.directory & "/" & name & " - " & type, true); + } + + // now convert back to the format returned by luceeVersionsListS3 + var versions=[]; + structEach( data, function( k,v ) { + arrayAppend( versions, v ); + }); + // systemOutput(serializeJson(var=data, compact=false), true); + + return versions; + } + +} \ No newline at end of file diff --git a/apps/updateserver/services/legacy/expressTemplates.cfc b/apps/updateserver/services/legacy/expressTemplates.cfc index dd9220d..fad8e59 100644 --- a/apps/updateserver/services/legacy/expressTemplates.cfc +++ b/apps/updateserver/services/legacy/expressTemplates.cfc @@ -33,8 +33,10 @@ component { arrayAppend( versions, ListLast( ListGetAt( s3_templates.name, 3, "-" ), "." ) ); } } - if ( arrayLen( versions ) eq 0 ) + if ( arrayLen( versions ) eq 0 ){ + systemOutput("express templates dir was empty? [#expressSrc#]", true); throw "getExpressTemplates() No express templates found for: [#tomcat_major#]"; + } ArraySort( versions, "numeric", "desc" ); // tomcat-9 = lucee-tomcat-9.0.100-template.zip express_templates[ ListFirst( listRest( tomcat_major, "-" ), "." ) ] @@ -64,7 +66,7 @@ component { private function _fetchExpressTemplateFromS3( name, src, dest ){ if ( !directoryExists( dest ) ) directoryCreate( dest ); - var _dest = getTempDirectory() & name; + var _dest = getTempDirectory( true ) & name; var _src = src & name; fileCopy( _src, _dest ); if ( isZipFile( _dest ) ){ diff --git a/compose.yaml b/compose.yaml index 9b6a551..a2b68aa 100644 --- a/compose.yaml +++ b/compose.yaml @@ -28,10 +28,13 @@ services: - 8888:8888 environment: - VIRTUAL_HOST=download.lucee.local - - S3_DOWNLOAD_ACCESS_KEY_ID=${S3_DOWNLOAD_ACCESS_KEY_ID:-xxxxxx} - - S3_DOWNLOAD_SECRET_KEY=${S3_DOWNLOAD_SECRET_KEY:-xxxxxx} + - UPDATE_PROVIDER=${UPDATE_PROVIDER} # defaults to prod, set UPDATE_PROVIDER=http://127.0.0.1:8889/rest/update/ in .env for local testing + - UPDATE_PROVIDER_INT=${UPDATE_PROVIDER_INT} # defaults to prod, set UPDATE_PROVIDER_INT=http://update:8888/rest/update/ in .env for local testing (docker) + - EXTENSION_PROVIDER=${EXTENSION_PROVIDER} # defaults to prod, set EXTENSION_PROVIDER=http://127.0.0.1:8889/rest/extension/ in .env for local testing + - EXTENSION_PROVIDER_INT=${EXTENSION_PROVIDER_INT} # defaults to prod, set EXTENSION_PROVIDER_INT=http://update:8888/rest/extension/ in .env for local testing (docker) - SENTRY_ENV=${SENTRY_ENV:-""} - SENTRY_DSN=${SENTRY_DSN:-""} + - DEBUG=${DEBUG:-"false"} update: build: @@ -45,10 +48,16 @@ services: - 8889:8888 environment: - VIRTUAL_HOST=update.lucee.local - - S3_EXTENSION_SECRET_KEY=${S3_EXTENSION_SECRET_KEY:-xxxxxx} - - S3_EXTENSION_ACCESS_KEY_ID=${S3_EXTENSION_ACCESS_KEY_ID:-xxxxxx} - - S3_DOWNLOAD_SECRET_KEY=${S3_DOWNLOAD_SECRET_KEY:-xxxxxx} - - S3_DOWNLOAD_ACCESS_KEY_ID=${S3_DOWNLOAD_ACCESS_KEY_ID:-xxxxxx} + - S3_EXTENSION_SECRET_KEY=${S3_EXTENSION_SECRET_KEY} + - S3_EXTENSION_ACCESS_KEY_ID=${S3_EXTENSION_ACCESS_KEY_ID} + # optionally use a local directory for testing + - S3_CORE_ROOT=${S3_CORE_ROOT} + - S3_EXTENSIONS_ROOT=${S3_EXTENSIONS_ROOT} + - S3_BUNDLES_ROOT=${S3_BUNDLES_ROOT} + - DOWNLOADS_URL=${DOWNLOADS_URL} + - UPDATE_PROVIDER=${UPDATE_PROVIDER} # defaults to prod, set UPDATE_PROVIDER=http://update:8888/rest/update in .env for local testing + - EXTENSION_PROVIDER=${EXTENSION_PROVIDER} # defaults to prod, set EXTENSION_PROVIDER=http://update:8888/rest/extension in .env for local testing - SENTRY_ENV=${SENTRY_ENV:-""} - SENTRY_DSN=${SENTRY_DSN:-""} - - ALLOW_RELOAD=${ALLOW_RELOAD:-"true"} \ No newline at end of file + - ALLOW_RELOAD=${ALLOW_RELOAD:-"true"} + - DEBUG=${DEBUG:-"false"} \ No newline at end of file diff --git a/devops/.CFconfig-download.json5 b/devops/.CFconfig-download.json5 index 74fbf54..62fe4a7 100644 --- a/devops/.CFconfig-download.json5 +++ b/devops/.CFconfig-download.json5 @@ -137,6 +137,12 @@ "level": "error", "layout": "classic" }, + "extension-provider": { + "appender": "resource", + "appenderArguments": "path:{lucee-config}/logs/extension-provider.log", + "level": "error", + "layout": "classic" + }, "exception": { "appender": "resource", "appenderArguments": "path:{lucee-config}/logs/exception.log", diff --git a/devops/.CFconfig-sentry.json b/devops/.CFconfig-sentry.json deleted file mode 100644 index b5716e2..0000000 --- a/devops/.CFconfig-sentry.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "loggers": { - "update-provider": { - "appenderArguments": "environment:%7Benv%2ESENTRY%5FENV%7D;dist:;extras:;dsn:%7Benv%2ESENTRY%5FDSN%7D;debug:false;tags", - "level": "error", - "appenderClass": "org.lucee.extension.sentry.log.log4j.SentryAppenderLog4j2", - "appenderBundleName": "sentry.extension", - "appenderBundleVersion": "5.5.2.15", - "layoutClass": "lucee.commons.io.log.log4j2.layout.ClassicLayout", - "layoutArguments": "" - }, - "exception": { - "appenderArguments": "environment:%7Benv%2ESENTRY%5FENV%7D;dist:;extras:;dsn:%7Benv%2ESENTRY%5FDSN%7D;debug:false;tags", - "level": "error", - "appenderClass": "org.lucee.extension.sentry.log.log4j.SentryAppenderLog4j2", - "appenderBundleName": "sentry.extension", - "appenderBundleVersion": "5.5.2.15", - "layoutClass": "lucee.commons.io.log.log4j2.layout.ClassicLayout", - "layoutArguments": "" - }, - "application": { - "appenderArguments": "environment:%7Benv%2ESENTRY%5FENV%7D;dist:;extras:;dsn:%7Benv%2ESENTRY%5FDSN%7D;debug:false;tags", - "level": "info", - "appenderClass": "org.lucee.extension.sentry.log.log4j.SentryAppenderLog4j2", - "appenderBundleName": "sentry.extension", - "appenderBundleVersion": "5.5.2.15", - "layoutClass": "lucee.commons.io.log.log4j2.layout.ClassicLayout", - "layoutArguments": "" - }, - "rest": { - "appenderArguments": "environment:%7Benv%2ESENTRY%5FENV%7D;dist:;extras:;dsn:%7Benv%2ESENTRY%5FDSN%7D;debug:false;tags", - "level": "error", - "appenderClass": "org.lucee.extension.sentry.log.log4j.SentryAppenderLog4j2", - "appenderBundleName": "sentry.extension", - "appenderBundleVersion": "5.5.2.15", - "layoutClass": "lucee.commons.io.log.log4j2.layout.ClassicLayout", - "layoutArguments": "" - } - } -} \ No newline at end of file diff --git a/devops/.CFconfig-update.json5 b/devops/.CFconfig-update.json5 index f264215..c7e156e 100644 --- a/devops/.CFconfig-update.json5 +++ b/devops/.CFconfig-update.json5 @@ -137,6 +137,12 @@ "level": "error", "layout": "classic" }, + "extension-provider": { + "appender": "resource", + "appenderArguments": "path:{lucee-config}/logs/extension-provider.log", + "level": "error", + "layout": "classic" + }, "exception": { "appender": "resource", "appenderArguments": "path:{lucee-config}/logs/exception.log", diff --git a/devops/Dockerfile.base b/devops/Dockerfile.base index 0bc6f5e..efd721b 100644 --- a/devops/Dockerfile.base +++ b/devops/Dockerfile.base @@ -1,15 +1,15 @@ #FROM lucee/lucee:6.0.3.1-tomcat9.0-jdk11-temurin-jammy -FROM lucee/lucee:6.2.0.321-light-nginx-tomcat10.1-jre21-temurin-jammy +FROM lucee/lucee:6.2.2.91-SNAPSHOT-light-nginx-tomcat11.0-jre21-temurin-noble ADD https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.20.0/jmx_prometheus_javaagent-0.20.0.jar /opt/lucee/bin/prometheus-java-agent.jar ENV JAVA_OPTS="-javaagent:/opt/lucee/bin/prometheus-java-agent.jar=9090:/opt/lucee/conf/prometheus/config.yml" ENV LUCEE_ADMIN_ENABLED=false RUN mkdir -p /opt/lucee/server/lucee-server/deploy && \ mkdir -p /opt/lucee/server/lucee-server/context && \ -wget -nv https://ext.lucee.org/sentry-extension-5.5.2.15.lex -O /opt/lucee/server/lucee-server/deploy/sentry-extension.lex && \ -wget -nv https://ext.lucee.org/s3-extension-2.0.2.21.lex -O /opt/lucee/server/lucee-server/deploy/s3-extension-2.0.2.21.lex && \ -wget -nv https://ext.lucee.org/image-extension-2.0.0.29.lex -O /opt/lucee/server/lucee-server/deploy/image-extension-2.0.0.29.lex && \ -wget -nv https://ext.lucee.org/esapi-extension-2.2.4.18.lex -O /opt/lucee/server/lucee-server/deploy/esapi-extension-2.2.4.18.lex && \ -wget -nv https://ext.lucee.org/compress-extension-1.0.0.15.lex -O /opt/lucee/server/lucee-server/deploy/compress-extension-1.0.0.15.lex +wget -nv https://ext.lucee.org/sentry-extension-5.6.0.0-SNAPSHOT.lex -O /opt/lucee/server/lucee-server/deploy/sentry-extension-5.6.0.0-SNAPSHOT.lex && \ +wget -nv https://ext.lucee.org/s3-extension-2.0.3.0.lex -O /opt/lucee/server/lucee-server/deploy/s3-extension-2.0.3.0.lex && \ +wget -nv https://ext.lucee.org/image-extension-2.0.0.31-SNAPSHOT.lex -O /opt/lucee/server/lucee-server/deploy/image-extension-2.0.0.31-SNAPSHOT.lex && \ +wget -nv https://ext.lucee.org/esapi-extension-2.6.0.1.lex -O /opt/lucee/server/lucee-server/deploy/esapi-extension-2.6.0.1.lex && \ +wget -nv https://ext.lucee.org/compress-extension-1.0.0.16.lex -O /opt/lucee/server/lucee-server/deploy/compress-extension-1.0.0.16.lex RUN mkdir -p /var/www && mkdir -p /var/www/logs \ && ln -sf /proc/1/fd/1 /opt/lucee/server/lucee-server/context/logs/application.log \ @@ -26,7 +26,8 @@ RUN mkdir -p /var/www && mkdir -p /var/www/logs \ && ln -sf /proc/1/fd/1 /opt/lucee/server/lucee-server/context/logs/scope.log \ && ln -sf /proc/1/fd/1 /opt/lucee/server/lucee-server/context/logs/search.log \ && ln -sf /proc/1/fd/1 /opt/lucee/server/lucee-server/context/logs/thread.log \ -&& ln -sf /proc/1/fd/1 /opt/lucee/server/lucee-server/context/logs/update-provider.log +&& ln -sf /proc/1/fd/1 /opt/lucee/server/lucee-server/context/logs/update-provider.log \ +&& ln -sf /proc/1/fd/1 /opt/lucee/server/lucee-server/context/logs/extension-provider.log #ENV LUCEE_LOGGING_FORCE_LEVEL=info #ENV LUCEE_LOGGING_FORCE_APPENDER=console diff --git a/devops/Dockerfile.download b/devops/Dockerfile.download index 14852d2..a7c0d8c 100644 --- a/devops/Dockerfile.download +++ b/devops/Dockerfile.download @@ -1,6 +1,4 @@ # syntax = edrevo/dockerfile-plus INCLUDE+ ./devops/Dockerfile.base COPY ./apps/download /var/www -COPY ./devops/.CFconfig-download.json5 /opt/lucee/server/lucee-server/context/.CFConfig.json -# post deploy configure sentry logs, once extension is installed -COPY ./devops/.CFconfig-sentry.json /opt/lucee/server/lucee-server/deploy/.CFconfig-sentry.json +COPY ./devops/.CFconfig-download.json5 /opt/lucee/server/lucee-server/context/.CFConfig.json \ No newline at end of file diff --git a/devops/Dockerfile.update b/devops/Dockerfile.update index d6be973..736b9ce 100644 --- a/devops/Dockerfile.update +++ b/devops/Dockerfile.update @@ -2,5 +2,4 @@ INCLUDE+ ./devops/Dockerfile.base COPY ./apps/updateserver /var/www COPY ./devops/.CFconfig-update.json5 /opt/lucee/server/lucee-server/context/.CFConfig.json -# post deploy configure sentry logs, once extension is installed -COPY ./devops/.CFconfig-sentry.json /opt/lucee/server/lucee-server/deploy/.CFconfig-sentry.json +COPY ./local_s3 /var/local_s3 diff --git a/local_s3/README.md b/local_s3/README.md new file mode 100644 index 0000000..a49a7cf --- /dev/null +++ b/local_s3/README.md @@ -0,0 +1,18 @@ +## Local folder instead of S3 + +create folders here for downloads, extensions, bundles + +add them to your `.env` file in the root directory of the project, to work and test locally + + +```env +S3_CORE_ROOT=/var/local_s3/lucee_downloads/ +S3_EXTENSIONS_ROOT=/var/local_s3/lucee_ext/ +S3_BUNDLES_ROOT=/var/local_s3/lucee_bundles/ +``` + +Under `local_s3` + +- create `lucee_downloads` and place a few sample Lucee full `.jar` files +- create `lucee_ext` place some sample extension `.lex` files +- under `lucee_downloads` create `express-templates` and download the files listed at https://update.lucee.org/rest/update/provider/expressTemplates into that dir \ No newline at end of file diff --git a/tests/testBuildArtifacts.cfc b/tests/testBuildArtifacts.cfc index b4d719c..866e686 100644 --- a/tests/testBuildArtifacts.cfc +++ b/tests/testBuildArtifacts.cfc @@ -13,18 +13,22 @@ component extends="org.lucee.cfml.test.LuceeTestCase" labels="data-provider-inte function run( testResults , testBox ) { describe( "test build artifacts", function() { + // javax, tomcat 9 + it(title="check valid artifacts are produced, 5.4.8.2", body=function(){ + buildArtifacts( "lucee-5.4.8.2" ); + }); // javax, tomcat 9 it(title="check valid artifacts are produced, 6.0.3.1", body=function(){ buildArtifacts( "lucee-6.0.3.1" ); }); // jakarta & javax, tomcat 10 - it(title="check valid artifacts are produced, 6.2.0.30-SNAPSHOT", body=function(){ - buildArtifacts( "lucee-6.2.0.30-SNAPSHOT" ); + it(title="check valid artifacts are produced, 6.2.2.91", body=function(){ + buildArtifacts( "lucee-6.2.2.91" ); }); // jakarta, tomcat 11 - it(title="check valid artifacts are produced, 7.0.0.159-SNAPSHOT", body=function(){ - buildArtifacts( "lucee-7.0.0.159-SNAPSHOT" ); + it(title="check valid artifacts are produced, 7.0.0.325-SNAPSHOT", body=function(){ + buildArtifacts( "lucee-7.0.0.325-SNAPSHOT" ); }); }); } diff --git a/tests/testJiraChangelog.cfc b/tests/testJiraChangelog.cfc index 13f25d3..d1703fa 100644 --- a/tests/testJiraChangelog.cfc +++ b/tests/testJiraChangelog.cfc @@ -90,12 +90,16 @@ component extends="org.lucee.cfml.test.LuceeTestCase" labels="data-provider" { return; } - // Find a ticket with a specific fix version to test with + // Find a ticket with a specific fix version to test with (skip versions with spaces like "Websocket-client 2.3.0.8") var testVersion = ""; loop query=issues { if ( isArray( issues.fixVersions ) && arrayLen( issues.fixVersions ) > 0 ) { - testVersion = issues.fixVersions[ 1 ]; - break; + var fv = issues.fixVersions[ 1 ]; + if ( isArray( fv ) ) fv = fv[ 1 ]; + if ( isSimpleValue( fv ) && !find( " ", fv ) ) { + testVersion = fv; + break; + } } } diff --git a/tests/testMavenMatcher.cfc b/tests/testMavenMatcher.cfc index 087a6ca..6166167 100644 --- a/tests/testMavenMatcher.cfc +++ b/tests/testMavenMatcher.cfc @@ -1,9 +1,9 @@ component extends="org.lucee.cfml.test.LuceeTestCase" labels="data-provider" { function beforeAll(){ - variables.root = getDirectoryFromPath(getCurrentTemplatePath()); - variables.root = listDeleteAt(root,listLen(root,"/\"), "/\") & "/"; // getDirectoryFromPath - variables.mavenMappingsFile = expandPath( "../../apps/updateserver/services/legacy/mavenMappings.json" ); + variables.root = getDirectoryFromPath( getCurrentTemplatePath() ); + variables.root = listDeleteAt( root, listLen( root, "/\" ), "/\" ) & "/"; + variables.mavenMappingsFile = variables.root & "apps/updateserver/services/legacy/mavenMappings.json"; }; function run( testResults , testBox ) { diff --git a/tests/testSentryLogger.cfc b/tests/testSentryLogger.cfc new file mode 100644 index 0000000..4675e27 --- /dev/null +++ b/tests/testSentryLogger.cfc @@ -0,0 +1,188 @@ +component extends="org.lucee.cfml.test.LuceeTestCase" labels="data-provider" { + + function beforeAll(){ + } + + function beforeAll (){ + variables.dir = getDirectoryFromPath(getCurrentTemplatePath()); + var servicesDir = expandPath( dir & "../apps/updateserver/services" ); + application action="update" mappings={ + "/services" : servicesDir + }; + // Use a fake DSN for testing + variables.testDsn = "https://test-public-key@test-host.sentry.io/12345"; + } + + function run( testResults, testBox ) { + describe( "SentryLogger Tests", function() { + + it( title="should initialize with valid DSN", body=function(){ + var logger = new services.SentryLogger( + config = { + dsn = testDsn, + environment = "test" + } + ); + expect( logger ).toBeInstanceOf( "services.SentryLogger" ); + }); + + it( title="should work with missing DSN (dev mode)", body=function(){ + var logger = new services.SentryLogger( + config = {} + ); + expect( logger ).toBeInstanceOf( "services.SentryLogger" ); + + // Should not actually send to Sentry but returns success + var result = logger.logMessage( + message = "Test message without DSN", + level = "info" + ); + expect( result ).toBeStruct(); + expect( result ).toHaveKey( "success" ); + expect( result.success ).toBeTrue(); // Returns true but doesn't send + expect( result ).toHaveKey( "note" ); // Should have a note explaining why + }); + + it( title="should work with empty DSN string", body=function(){ + var logger = new services.SentryLogger( + config = { dsn = "" } + ); + expect( logger ).toBeInstanceOf( "services.SentryLogger" ); + + var result = logger.logMessage( + message = "Test message with empty DSN", + level = "info" + ); + expect( result ).toBeStruct(); + expect( result.success ).toBeTrue(); // Returns true but doesn't send + expect( result ).toHaveKey( "note" ); // Should explain no DSN + }); + + it( title="should fail with invalid DSN format", body=function(){ + expect( function(){ + new services.SentryLogger( + config = { dsn = "invalid-dsn-format" } + ); + }).toThrow( type="Sentry.ConfigurationException" ); + }); + + it( title="should parse DSN correctly", body=function(){ + var logger = new services.SentryLogger( + config = { dsn = testDsn } + ); + + // Access private sentryInfo via reflection or test public behavior + var result = logger.logMessage( + message = "Test message", + level = "info" + ); + + // Should return a struct with success/error info + expect( result ).toBeStruct(); + expect( result ).toHaveKey( "success" ); + }); + + it( title="should log message without throwing", body=function(){ + var logger = new services.SentryLogger( + config = { + dsn = testDsn, + environment = "test" + } + ); + + var result = logger.logMessage( + message = "Test info message", + level = "info" + ); + + expect( result ).toBeStruct(); + }); + + it( title="should log exception without throwing", body=function(){ + var logger = new services.SentryLogger( + config = { + dsn = testDsn, + environment = "test" + } + ); + + try { + throw( type="TestException", message="This is a test exception" ); + } catch ( any e ) { + var result = logger.logException( + exception = e, + level = "error" + ); + + expect( result ).toBeStruct(); + expect( result ).toHaveKey( "success" ); + } + }); + + it( title="should accept tags and extra data", body=function(){ + var logger = new services.SentryLogger( + config = { + dsn = testDsn, + environment = "test" + } + ); + + var result = logger.logMessage( + message = "Test with metadata", + level = "info", + tags = { "component" = "test", "action" = "testing" }, + extra = { "userId" = 123, "debugInfo" = "some data" } + ); + + expect( result ).toBeStruct(); + }); + + it( title="should accept user context", body=function(){ + var logger = new services.SentryLogger( + config = { + dsn = testDsn, + environment = "test" + } + ); + + var result = logger.logMessage( + message = "Test with user", + level = "info", + user = { "id" = "user123", "email" = "test@example.com" } + ); + + expect( result ).toBeStruct(); + }); + + it( title="should use default environment if not specified", body=function(){ + var logger = new services.SentryLogger( + config = { dsn = testDsn } + ); + + var result = logger.logMessage( + message = "Test default environment", + level = "info" + ); + + expect( result ).toBeStruct(); + }); + + it( title="should handle different severity levels", body=function(){ + var logger = new services.SentryLogger( + config = { dsn = testDsn } + ); + + var levels = [ "debug", "info", "warning", "error", "fatal" ]; + + for ( var level in levels ) { + var result = logger.logMessage( + message = "Test #level# level", + level = level + ); + expect( result ).toBeStruct(); + } + }); + + }); + } +} diff --git a/tests/testStableReleaseBundles.cfc b/tests/testStableReleaseBundles.cfc index 2f0fa27..3a5bdfb 100644 --- a/tests/testStableReleaseBundles.cfc +++ b/tests/testStableReleaseBundles.cfc @@ -12,6 +12,10 @@ component extends="org.lucee.cfml.test.LuceeTestCase" labels="data-provider-inte function run( testResults , testBox ) { describe( "check all bundles in stable lucee release manifests are supported", function() { + it(title="6.2.2.91", body=function(){ + checkRequiredBundlesAreSupported( "6.2.2.91" ); + }); + it(title="6.0.3.1", body=function(){ checkRequiredBundlesAreSupported( "6.0.3.1" ); });