diff --git a/.github/workflows/build-prod.yml b/.github/workflows/build-prod.yml index 5b893426..e694fd62 100644 --- a/.github/workflows/build-prod.yml +++ b/.github/workflows/build-prod.yml @@ -59,14 +59,6 @@ jobs: env: NODE_ENV: production KG_AUTH: ${{ secrets.KG_AUTH }} - continue-on-error: false - - - name: Check for KG fallback usage - run: | - if grep -q "Using fallback data" web/_site/index.html; then - echo "WARNING: Using fallback KG data" >> $GITHUB_STEP_SUMMARY - echo "::warning::Using fallback KG data" - fi - name: Verify build run: ls -la ./_site @@ -79,22 +71,6 @@ jobs: sudo chmod 600 ~/.ssh/do_uxm_prod ssh-keyscan -H "24.199.98.130" > ~/.ssh/known_hosts - - name: Manage fallback data directory - run: | - # Create directory if it doesn't exist - ssh -i ~/.ssh/do_uxm_prod prod@24.199.98.130 "mkdir -p /var/www/uxm/fallback-data" - - # Keep only the 2 most recent backup files - ssh -i ~/.ssh/do_uxm_prod prod@24.199.98.130 "ls -t /var/www/uxm/fallback-data/kg-data-*.json | tail -n +3 | xargs -r rm" - - - name: Upload fallback data - run: | - if [ -f "web/_data/fallback-data/kg-data.json" ]; then - # Add timestamp to filename to keep history - TIMESTAMP=$(date +%Y%m%d-%H%M%S) - scp -i ~/.ssh/do_uxm_prod web/_data/fallback-data/kg-data.json prod@24.199.98.130:/var/www/uxm/fallback-data/kg-data-${TIMESTAMP}.json - fi - - name: Clean & FTP Upload run: | ssh -i ~/.ssh/do_uxm_prod prod@24.199.98.130 "rm -rf /var/www/uxm/html/*" diff --git a/web/.eleventy.js b/web/.eleventy.js index f5caf382..3dbcd4e5 100644 --- a/web/.eleventy.js +++ b/web/.eleventy.js @@ -30,7 +30,6 @@ export default function (eleventyConfig) { // Watch all asset directories for changes eleventyConfig.addWatchTarget('_src/**/*') // Watch everything in _src - eleventyConfig.watchIgnores.add('_data/fallback-data/**/*') // Ignore fallback data directory eleventyConfig.setServerOptions({ showAllHosts: true, diff --git a/web/_data/methods.js b/web/_data/methods.js index 6683fcf7..6441b147 100644 --- a/web/_data/methods.js +++ b/web/_data/methods.js @@ -1,71 +1,16 @@ import {client} from '../utils/sanityClient.js' import {toHTML} from '@portabletext/to-html' import groq from 'groq' -import {readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync, statSync, unlinkSync} from 'fs' -import {join} from 'path' -import * as dotenv from 'dotenv' -// import { uxmComponents } from '../utils/serializers.js' +import dotenv from 'dotenv' // Load environment variables from .env.local in development only if (process.env.NODE_ENV !== 'production') { dotenv.config({ path: '.env.local' }) } -// Constants for fallback data -const FALLBACK_DIR = process.env.NODE_ENV === 'production' - ? '/var/www/uxm/fallback-data' - : join(process.cwd(), '_data/fallback-data') -const FALLBACK_FILE = join(FALLBACK_DIR, 'kg-data.json') - -// Clean up old fallback files in development -function cleanupFallbackData() { - if (process.env.NODE_ENV !== 'production') { - try { - if (existsSync(FALLBACK_DIR)) { - // Get all fallback files sorted by modification time - const files = readdirSync(FALLBACK_DIR) - .filter(file => file.startsWith('kg-data-') && file.endsWith('.json')) - .map(file => ({ - name: file, - path: join(FALLBACK_DIR, file), - time: statSync(join(FALLBACK_DIR, file)).mtime.getTime() - })) - .sort((a, b) => b.time - a.time) - - // Keep only the 2 most recent files - files.slice(2).forEach(file => { - unlinkSync(file.path) - console.log(`Removed old fallback file: ${file.name}`) - }) - } - } catch (error) { - console.warn('Error cleaning up fallback data:', error.message) - } - } -} - -// Get the most recent fallback file -function getMostRecentFallback() { - try { - if (existsSync(FALLBACK_DIR)) { - const files = readdirSync(FALLBACK_DIR) - .filter(file => file.startsWith('kg-data-') && file.endsWith('.json')) - .map(file => ({ - name: file, - path: join(FALLBACK_DIR, file), - time: statSync(join(FALLBACK_DIR, file)).mtime.getTime() - })) - .sort((a, b) => b.time - a.time) - - if (files.length > 0) { - return files[0].path - } - } - return null - } catch (error) { - console.warn('Error finding most recent fallback:', error.message) - return null - } +// Ensure required environment variables are set +if (!process.env.KG_AUTH) { + throw new Error('KG_AUTH environment variable is required') } // Transform SPARQL results into a more usable format @@ -78,77 +23,36 @@ function transformKgData(kgData) { })) } -// Get shared output from KG with fallback mechanism +// Get shared output from KG async function getSharedOutput() { - try { - const kgData = await fetch('http://kg.uxmethods.org/repositories/uxm', { - method: 'POST', - headers: { - 'Content-Type': 'application/sparql-query', - Accept: 'application/sparql-results+json', - Authorization: 'Basic ' + btoa(process.env.KG_AUTH), - }, - body: ` - PREFIX : - PREFIX uxmo: - - SELECT ?origin ?destination (COUNT(?output) AS ?sharedOutputCount) - (GROUP_CONCAT(DISTINCT ?output; SEPARATOR=",") AS ?sharedOutput) - WHERE { - ?origin uxmo:hasOutput ?output. - ?destination uxmo:hasInput ?output. - } - GROUP BY ?origin ?destination - ORDER BY DESC(?sharedOutputCount) - `, - }) - - if (!kgData.ok) { - throw new Error(`HTTP error! status: ${kgData.status}`) - } - - const data = await kgData.json() - const transformedData = transformKgData(data) - - // Store successful response as backup - const backupData = { - timestamp: new Date().toISOString(), - data: transformedData - } - - // Only create directory and write files in development - if (process.env.NODE_ENV !== 'production') { - // Ensure fallback directory exists - if (!existsSync(FALLBACK_DIR)) { - mkdirSync(FALLBACK_DIR, { recursive: true }) - } - - const timestamp = new Date().toISOString().replace(/[:.]/g, '-') - const backupFile = join(FALLBACK_DIR, `kg-data-${timestamp}.json`) - writeFileSync(backupFile, JSON.stringify(backupData, null, 2)) - cleanupFallbackData() - } - - return transformedData - } catch (error) { - console.warn('Failed to fetch from KG, attempting to use fallback data:', error.message) - - try { - // Try to read fallback data - const fallbackPath = process.env.NODE_ENV === 'production' - ? FALLBACK_FILE - : getMostRecentFallback() + const kgData = await fetch('http://kg.uxmethods.org/repositories/uxm', { + method: 'POST', + headers: { + 'Content-Type': 'application/sparql-query', + Accept: 'application/sparql-results+json', + Authorization: 'Basic ' + btoa(process.env.KG_AUTH), + }, + body: ` + PREFIX : + PREFIX uxmo: + + SELECT ?origin ?destination (COUNT(?output) AS ?sharedOutputCount) + (GROUP_CONCAT(DISTINCT ?output; SEPARATOR=",") AS ?sharedOutput) + WHERE { + ?origin uxmo:hasOutput ?output. + ?destination uxmo:hasInput ?output. + } + GROUP BY ?origin ?destination + ORDER BY DESC(?sharedOutputCount) + `, + }) - if (fallbackPath && existsSync(fallbackPath)) { - const fallbackData = JSON.parse(readFileSync(fallbackPath, 'utf-8')) - console.warn('Using fallback data from:', fallbackData.timestamp) - return fallbackData.data - } - throw new Error('No fallback data available') - } catch (fallbackError) { - throw new Error(`Both KG fetch and fallback failed: ${fallbackError.message}`) - } + if (!kgData.ok) { + throw new Error(`Failed to fetch from KG: HTTP error! status: ${kgData.status}`) } + + const data = await kgData.json() + return transformKgData(data) } function prepareMethod(methods, methodPreviews, kgData) { @@ -186,36 +90,36 @@ async function getMethods() { const [methods, kgData] = await Promise.all([ client.fetch(groq` *[_type == "method"] | order(title) { - title, - "type": "method", - "slug": slug.current, - "uri": uri.current, - "createdAt": dateStamp.createdAt, - "revisedAt": dateStamp.revisedAt, - metaDescription, - "heroImage": { - "credit": heroImage.asset->creditLine, - "source": heroImage.asset->source.url, - "url": heroImage.asset->url, - ...heroImage - }, - overview, - steps, - stepSources, - dateStamps, - "outcomes": output[]->{ - prefLabel, - definition, - }, - "resources": *[_type == "resource" && references(^._id)]{ title, - author, - resourceUrl, - resourceImage, - "publisher": publisher.pubName + "type": "method", + "slug": slug.current, + "uri": uri.current, + "createdAt": dateStamp.createdAt, + "revisedAt": dateStamp.revisedAt, + metaDescription, + "heroImage": { + "credit": heroImage.asset->creditLine, + "source": heroImage.asset->source.url, + "url": heroImage.asset->url, + ...heroImage + }, + overview, + steps, + stepSources, + dateStamps, + "outcomes": output[]->{ + prefLabel, + definition, + }, + "resources": *[_type == "resource" && references(^._id)]{ + title, + author, + resourceUrl, + resourceImage, + "publisher": publisher.pubName + } } - } - `), + `), getSharedOutput() ])