Skip to content

Add scan-to-scan accessibility regression detection and sidebar trend messaging #1588

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
SteveJonesDev wants to merge 2 commits into
base: develop
Choose a base branch
from
Open

Add scan-to-scan accessibility regression detection and sidebar trend messaging #1588

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0638fd8
Add scan-to-scan accessibility regression detection
SteveJonesDev Mar 20, 2026
acc8794
Enhance regression detection: preserve existing data when issue count…
SteveJonesDev Mar 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions includes/classes/class-rest-api.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -874,6 +874,29 @@ private function get_summary_data( $post_id ) {
'warnings' => 0,
'ignored' => 0,
'readability' => 0,
'regression' => [
'has_baseline' => false,
'status' => 'stable',
'delta' => [
'errors' => 0,
'warnings' => 0,
'contrast_errors' => 0,
'passed_tests' => 0,
],
'scanned_at' => '',
],
];
} elseif ( ! isset( $summary['regression'] ) || ! is_array( $summary['regression'] ) ) {
$summary['regression'] = [
'has_baseline' => false,
'status' => 'stable',
'delta' => [
'errors' => 0,
'warnings' => 0,
'contrast_errors' => 0,
'passed_tests' => 0,
],
'scanned_at' => '',
];
}

Expand Down
146 changes: 144 additions & 2 deletions includes/classes/class-summary-generator.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,9 @@ public function __construct( $post_id ) {
public function generate_summary() {
global $wpdb;

$table_name = edac_get_valid_table_name( $wpdb->prefix . 'accessibility_checker' );
$summary = [];
$table_name = edac_get_valid_table_name( $wpdb->prefix . 'accessibility_checker' );
$summary = [];
$previous_summary = get_post_meta( $this->post_id, '_edac_summary', true );

if ( ! $table_name ) {
return $summary;
Expand All @@ -80,12 +81,96 @@ public function generate_summary() {
$summary['content_grade'] = $this->calculate_content_grade();
$summary['readability'] = $this->get_readability( $summary );
$summary['simplified_summary'] = (bool) ( get_post_meta( $this->post_id, '_edac_simplified_summary', true ) );

// If issue counts haven't changed since the last save, preserve existing regression
// data to avoid a second generate_summary() call overwriting deltas with zeros.
if ( $this->issue_counts_match( $previous_summary, $summary ) && ! empty( $previous_summary['regression'] ) ) {
$summary['regression'] = $previous_summary['regression'];
} else {
$summary['regression'] = $this->build_regression_data( $previous_summary, $summary );
}

$this->update_issue_density( $summary );
$this->save_summary_meta_data( $summary );

return $summary;
}

/**
* Checks whether issue-related counts in the current summary match a previously saved summary.
*
* @param mixed $previous_summary The previously saved summary meta.
* @param array $current_summary The current summary data.
*
* @return bool True when all issue counts are identical.
*/
private function issue_counts_match( $previous_summary, array $current_summary ): bool {
if ( ! is_array( $previous_summary ) ) {
return false;
}

$metrics = [ 'errors', 'warnings', 'contrast_errors', 'passed_tests', 'content_grade' ];

foreach ( $metrics as $metric ) {
if ( absint( $previous_summary[ $metric ] ?? 0 ) !== absint( $current_summary[ $metric ] ?? 0 ) ) {
return false;
}
}

return true;
}

/**
* Build regression/trend data by comparing the current summary with the previous scan summary.
*
* @param mixed $previous_summary The previously saved summary meta.
* @param array $current_summary The current summary data.
*
* @return array
*/
private function build_regression_data( $previous_summary, array $current_summary ) {
$metrics = [
'errors',
'warnings',
'contrast_errors',
'passed_tests',
'content_grade',
];

$deltas = [];
foreach ( $metrics as $metric ) {
$current_value = absint( $current_summary[ $metric ] ?? 0 );
$previous_value = is_array( $previous_summary ) ? absint( $previous_summary[ $metric ] ?? 0 ) : 0;
$deltas[ $metric ] = $current_value - $previous_value;
}

$regression_score = max( 0, $deltas['errors'] ) + max( 0, $deltas['warnings'] ) + max( 0, $deltas['contrast_errors'] );

$status = 'stable';
if ( $regression_score >= 5 || $deltas['passed_tests'] <= -10 ) {
$status = 'declining';
} elseif ( $regression_score > 0 || $deltas['passed_tests'] < 0 ) {
$status = 'watch';
} elseif ( $deltas['errors'] < 0 || $deltas['warnings'] < 0 || $deltas['contrast_errors'] < 0 || $deltas['passed_tests'] > 0 ) {
Comment on lines +150 to +154
Copy link
Copy Markdown

@chatgpt-codex-connector chatgpt-codex-connector Bot Mar 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Gate regression status on baseline availability

This classification runs even when there is no prior summary, so a first-ever scan with existing errors/warnings is labeled watch/declining despite has_baseline being false. The sidebar then uses that status for retention messaging, which can incorrectly tell users accessibility is declining before any scan-to-scan comparison exists. Please short-circuit to stable (or equivalent neutral status) until a baseline is present.

Useful? React with 👍 / 👎.

$status = 'improving';
}

$has_baseline = is_array( $previous_summary ) && ! empty( $previous_summary );

return [
'has_baseline' => (bool) $has_baseline,
'status' => sanitize_key( $status ),
'delta' => [
'errors' => intval( $deltas['errors'] ),
'warnings' => intval( $deltas['warnings'] ),
'contrast_errors' => intval( $deltas['contrast_errors'] ),
'passed_tests' => intval( $deltas['passed_tests'] ),
'content_grade' => intval( $deltas['content_grade'] ),
],
'scanned_at' => current_time( 'mysql', true ),
];
}

/**
* Calculates the percentage of passed tests based on the provided rules.
* This method queries the database to find which rules have not been violated (passed) for
Expand Down Expand Up @@ -330,6 +415,51 @@ private function save_summary_meta_data( $summary ) {
update_post_meta( $this->post_id, '_edac_summary_warnings', absint( $summary['warnings'] ) );
update_post_meta( $this->post_id, '_edac_summary_ignored', absint( $summary['ignored'] ) );
update_post_meta( $this->post_id, '_edac_summary_contrast_errors', absint( $summary['contrast_errors'] ) );
$this->update_summary_history( $summary );
}

/**
* Stores compact summary scan history used for trend/regression messaging.
*
* @param array $summary The latest summary.
*
* @return void
*/
private function update_summary_history( array $summary ) {
$history = get_post_meta( $this->post_id, '_edac_summary_history', true );
$history = is_array( $history ) ? $history : [];
$timestamp = current_time( 'mysql', true );

$new_entry = [
'scanned_at' => $timestamp,
'errors' => absint( $summary['errors'] ?? 0 ),
'warnings' => absint( $summary['warnings'] ?? 0 ),
'contrast_errors' => absint( $summary['contrast_errors'] ?? 0 ),
'passed_tests' => absint( $summary['passed_tests'] ?? 0 ),
'content_grade' => absint( $summary['content_grade'] ?? 0 ),
];

// Skip duplicate entries when generate_summary() is called multiple times per scan.
if ( ! empty( $history ) ) {
$last = end( $history );
if (
absint( $last['errors'] ?? -1 ) === $new_entry['errors'] &&
absint( $last['warnings'] ?? -1 ) === $new_entry['warnings'] &&
absint( $last['contrast_errors'] ?? -1 ) === $new_entry['contrast_errors'] &&
absint( $last['passed_tests'] ?? -1 ) === $new_entry['passed_tests'] &&
absint( $last['content_grade'] ?? -1 ) === $new_entry['content_grade']
) {
return;
}
Comment on lines +443 to +453
Copy link
Copy Markdown
Contributor

@coderabbitai coderabbitai Bot Mar 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

Default value with absint() can cause false matches.

Using -1 as a fallback with absint() yields 1, which could incorrectly match a legitimate entry where the metric equals 1—causing that entry to be skipped.

Consider using a direct comparison without the absint() wrapper on the fallback, or restructure to avoid the ambiguity:

🛡️ Suggested fix 
 		if (
-			absint( $last['errors'] ?? -1 ) === $new_entry['errors'] &&
-			absint( $last['warnings'] ?? -1 ) === $new_entry['warnings'] &&
-			absint( $last['contrast_errors'] ?? -1 ) === $new_entry['contrast_errors'] &&
-			absint( $last['passed_tests'] ?? -1 ) === $new_entry['passed_tests'] &&
-			absint( $last['content_grade'] ?? -1 ) === $new_entry['content_grade']
+			isset( $last['errors'], $last['warnings'], $last['contrast_errors'], $last['passed_tests'], $last['content_grade'] ) &&
+			absint( $last['errors'] ) === $new_entry['errors'] &&
+			absint( $last['warnings'] ) === $new_entry['warnings'] &&
+			absint( $last['contrast_errors'] ) === $new_entry['contrast_errors'] &&
+			absint( $last['passed_tests'] ) === $new_entry['passed_tests'] &&
+			absint( $last['content_grade'] ) === $new_entry['content_grade']
 		) {
🤖 Prompt for AI Agents 
Verify each finding against the current code and only fix it if needed.

In `@includes/classes/class-summary-generator.php` around lines 443 - 453, The
current comparison uses absint(... ?? -1) which turns a missing value into 1 and
can falsely match real values; update the block that checks $last vs $new_entry
so you first ensure each metric exists (use isset($last['errors']) /
isset($new_entry['errors']) etc.) and then compare their integer values (e.g.,
absint($last['errors']) === absint($new_entry['errors'])) only when both exist,
otherwise treat as different; apply this pattern to errors, warnings,
contrast_errors, passed_tests and content_grade to avoid the -1 fallback
ambiguity.

}

$history[] = $new_entry;

if ( count( $history ) > 10 ) {
$history = array_slice( $history, -10 );
}

update_post_meta( $this->post_id, '_edac_summary_history', $history );
}

/**
Expand All @@ -351,6 +481,18 @@ private function sanitize_summary_meta_data( array $summary ): array {
'content_grade' => absint( $summary['content_grade'] ?? 0 ),
'readability' => sanitize_text_field( $summary['readability'] ?? '' ),
'simplified_summary' => filter_var( $summary['simplified_summary'] ?? false, FILTER_VALIDATE_BOOLEAN ),
'regression' => [
'has_baseline' => filter_var( $summary['regression']['has_baseline'] ?? false, FILTER_VALIDATE_BOOLEAN ),
'status' => sanitize_key( $summary['regression']['status'] ?? 'stable' ),
'delta' => [
'errors' => intval( $summary['regression']['delta']['errors'] ?? 0 ),
'warnings' => intval( $summary['regression']['delta']['warnings'] ?? 0 ),
'contrast_errors' => intval( $summary['regression']['delta']['contrast_errors'] ?? 0 ),
'passed_tests' => intval( $summary['regression']['delta']['passed_tests'] ?? 0 ),
'content_grade' => intval( $summary['regression']['delta']['content_grade'] ?? 0 ),
],
'scanned_at' => sanitize_text_field( $summary['regression']['scanned_at'] ?? '' ),
],
];
}
}
4 changes: 2 additions & 2 deletions package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

72 changes: 63 additions & 9 deletions src/sidebar/components/Panels/AccessibilityStatus.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ const AccessibilityStatus = () => {
// Extract data from store
const summary = data?.summary || {};
const readability = data?.readability || {};
const regression = summary.regression || {};
const regressionDelta = regression.delta || {};

const coveragePercent = summary.passed_tests || 0;
const problems = ( summary.errors || 0 ) + ( summary.contrast_errors || 0 );
Expand All @@ -78,6 +80,47 @@ const AccessibilityStatus = () => {
const postGradeReadable = readability.post_grade_readability || '';
const needsSummary = !! readability.post_grade_failed;
const hasSummary = !! readability.simplified_summary;
const hasRegressionBaseline = !! regression.has_baseline;
const regressionStatus = regression.status || 'stable';
const problemsDelta =
( regressionDelta.errors || 0 ) + ( regressionDelta.contrast_errors || 0 );
const warningDelta = regressionDelta.warnings || 0;
const passedDelta = regressionDelta.passed_tests || 0;
const readingLevelDelta = regressionDelta.content_grade || 0;

const renderRegressionMessage = useCallback(
( delta, noun, invertMeaning = false ) => {
if ( ! hasRegressionBaseline ) {
return __( 'Run another scan to start tracking trend changes.', 'accessibility-checker' );
}

if ( delta === 0 ) {
return sprintf( __( 'No change in %s since the last scan.', 'accessibility-checker' ), noun );
}

const increase = delta > 0;
const absDelta = Math.abs( delta );
const directionWord = increase ? __( 'up', 'accessibility-checker' ) : __( 'down', 'accessibility-checker' );
const impactWord = ( increase && ! invertMeaning ) || ( ! increase && invertMeaning )
? __( 'Regression', 'accessibility-checker' )
: __( 'Improvement', 'accessibility-checker' );

return sprintf(
// translators: 1: regression/improvement label, 2: magnitude, 3: metric label, 4: up/down.
__( '%1$s: %2$d %3$s %4$s since last scan.', 'accessibility-checker' ),
impactWord,
absDelta,
noun,
directionWord,
);
},
[ hasRegressionBaseline ],
);

const retentionMessage =
regressionStatus === 'declining'
? __( 'Accessibility is declining. If you stop scanning, this drift may go unnoticed until users are blocked.', 'accessibility-checker' )
: '';

// Determine status icon based on errors and warnings
let statusIconName = 'check';
Expand Down Expand Up @@ -206,7 +249,9 @@ const AccessibilityStatus = () => {
<div className="edac-status-card__value">
{ problems }
</div>
{/* Placeholder for 30-day trend - will be implemented later */}
<div className="edac-status-card__meta edac-status-card__meta--trend">
{ renderRegressionMessage( problemsDelta, __( 'problems', 'accessibility-checker' ) ) }
</div>
</div>

{/* Needs Review (Warnings) */}
Expand Down Expand Up @@ -240,7 +285,9 @@ const AccessibilityStatus = () => {
<div className="edac-status-card__value">
{ needsReview }
</div>
{/* Placeholder for 30-day trend - will be implemented later */}
<div className="edac-status-card__meta edac-status-card__meta--trend">
{ renderRegressionMessage( warningDelta, __( 'warnings', 'accessibility-checker' ) ) }
</div>
</div>
{/* Reading Level */}
<div
Expand Down Expand Up @@ -273,11 +320,12 @@ const AccessibilityStatus = () => {
<div className="edac-status-card__value">
{ readingLevelText }
</div>
{ summaryStatus && (
<div className="edac-status-card__meta">
{ summaryStatus }
</div>
) }
<div className="edac-status-card__meta edac-status-card__meta--trend">
{ summaryStatus && (
<span>{ summaryStatus }. </span>
) }
{ renderRegressionMessage( readingLevelDelta, __( 'reading level', 'accessibility-checker' ) ) }
</div>
</div>
{/* Passed Checks (Coverage) */}
<div className="edac-status-card">
Expand All @@ -290,13 +338,19 @@ const AccessibilityStatus = () => {
<div className="edac-status-card__value">
{ coveragePercent }%
</div>
{/* Placeholder for 30-day trend - will be implemented later */}
<div className="edac-status-card__meta edac-status-card__meta--trend">
{ renderRegressionMessage( passedDelta, __( 'passed checks', 'accessibility-checker' ), true ) }
</div>
</div>
</PanelRow>
{ retentionMessage && (
<PanelRow className="edac-status-retention-message">
<p>{ retentionMessage }</p>
</PanelRow>
) }
</PanelBody>
</Panel>
);
};

export default AccessibilityStatus;

20 changes: 19 additions & 1 deletion src/sidebar/sass/components/accessibility-status.scss
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,25 @@
font-size: 10px;
color: $info-blue;
margin-top: 2px;

&--trend {
line-height: 1.35;
min-height: 28px;
color: $subtext-gray;
}
}
}

.edac-status-retention-message {
margin: 10px 0 0;
padding-top: 10px;
border-top: 1px dashed $outline-grey;

p {
margin: 0;
font-size: 11px;
line-height: 1.4;
color: $info-grey;
}
}

Expand All @@ -126,4 +145,3 @@
}
}
}

Loading
Loading