Infrastructure for individual entity files

Author: André L F S Bacci

.gitignore

@@ -2,8 +2,6 @@
 .manual.xml
 .entities.ent
 .revcheck.json
-install-unix.xml
-install-win.xml
 manual.xml
 version.xml
 sources.xml

configure.php

@@ -739,11 +739,12 @@ function getFileModificationHistory(): array {
 globbetyglob("{$ac['basedir']}/scripts", 'make_scripts_executable');
 
 {
-    $cmd  = escapeshellarg( $ac['PHP'] );
-    $cmd .= ' ' . escapeshellarg( __DIR__ . '/scripts/entities.php' );
-    $cmd .= ' ' . escapeshellarg( $ac['ROOTDIR'] . '/en/entities' );
+    $cmd[] = escapeshellarg( $ac['PHP'] );
+    $cmd[] = escapeshellarg( __DIR__ . '/scripts/entities.php' );
+    $cmd[] = escapeshellarg( $ac['ROOTDIR'] . '/en/entities' );
     if ( $ac['LANG'] != 'en' )
-        $cmd .= ' ' . escapeshellarg( $ac['ROOTDIR'] . '/' . $ac['LANG'] . '/entities' );
+        $cmd[] = escapeshellarg( $ac['ROOTDIR'] . '/' . $ac['LANG'] . '/entities' );
+    $cmd = implode( ' ' , $cmd );
     passthru( $cmd );
 }
 

scripts/entities.php

@@ -33,7 +33,10 @@
     return;
 }
 
-$generate = realpath( __DIR__ . "/../.entities.ent" );  // sibling of .manual.xml
+$filename = __DIR__ . "/../.entities.ent";  // sibling of .manual.xml
+touch( $filename );                         // empty file, at minimum, and
+$filename = realpath( $filename );          // realpath() fails if file not exists.
+
 $entities = []; // all entitites, already overriden
 $expected = []; // entities that are expected to be oversidem (translatins)
 $override = []; // overrides stattics
@@ -48,16 +51,14 @@
         $langs[] = $argv[$idx];
 
 if ( $detail )
-    print "Creating file $generate in verbose detail mode...\n";
+    print "Creating file $filename in verbose detail mode...\n";
 else
-    print "Creating file $generate...";
+    print "Creating file $filename...";
 
 for ( $run = 0 ; $run < count( $langs) ; $run++ )
     parseDir( $langs[$run] , $run > 0 );
 
-dump( $entities );
-
-[$count, $untranslated, $overriden] = verifyOverrides( $detail );
+dump( $filename , $entities );
 
 if ( $detail )
 {
@@ -66,8 +67,9 @@
 else
 {
     echo " done";
-    if ( $untranslated + $overriden > 0 )
-        echo ": $count entities, $untranslated untranslated, $overriden orerriden";
+    [$all, $unt, $over] = verifyOverrides( $detail );
+    if ( $unt + $over > 0 )
+        echo ": $all entities, $unt untranslated, $over orerriden";
     echo ".\n";
 }
 exit;
@@ -92,52 +94,59 @@ function parseDir( string $dir , bool $expectedOverride )
             continue;
 
         $text = file_get_contents( $path );
-
-        if ( validate( $path , $text , false ) )
-        {
-            push( $path , $text , $expectedOverride );
-            continue;
-        }
-
-        $frag = "<frag>$text</frag>";
-        if ( validate( $path , $frag , true ) )
-        {
-            push( $path , $text , $expectedOverride );
-            continue;
-        }
+        validateStore( $path , $text , $expectedOverride );
     }
 }
 
-function validate( string $path , string $text , bool $warn ) : bool
+function validateStore( string $path , string $text , bool $expectedOverride )
 {
+    $trim = trim( $text );
+    if ( strlen( $trim ) == 0 )
+    {
+        // Yes, there is empty entities, and they are valid entity, but not valid XML.
+        // see: en/language-snippets.ent mongodb.note.queryable-encryption-preview
+        push( $path , $text , $expectedOverride , true );
+        return;
+    }
+
+    $frag = "<root>$text</root>";
+
     $dom = new DOMDocument( '1.0' , 'utf8' );
     $dom->recover = true;
     $dom->resolveExternals = false;
     libxml_use_internal_errors( true );
 
-    $res = $dom->loadXML( $text );
+    $res = $dom->loadXML( $frag );
 
     $err = libxml_get_errors();
     libxml_clear_errors();
 
     foreach( $err as $item )
     {
-        $msg = $item->message;
+        $msg = trim( $item->message );
         if ( str_starts_with( $msg , "Entity '" ) && str_ends_with( $msg , "' not defined" ) )
             continue;
 
-        if ( $warn )
-        {
-            fwrite( STDERR , "\n  XML load failed on entity file." );
-            fwrite( STDERR , "\n    Path:  $path" );
-            fwrite( STDERR , "\n    Error: $msg\n" );
-        }
-        return false;
+        fwrite( STDERR , "\n  XML load failed on entity file." );
+        fwrite( STDERR , "\n    Path:  $path" );
+        fwrite( STDERR , "\n    Error: $msg\n" );
+        return;
     }
-    return true;
+
+    $inline = shouldInline( $dom );
+    push( $path , $text , $expectedOverride , $inline );
 }
 
-function push( string $path , string $contents , bool $expectedOverride )
+class EntityData
+{
+    public function __construct(
+        public string $path ,
+        public string $name ,
+        public string $text ,
+        public bool $inline ) {}
+}
+
+function push( string $path , string $text , bool $expectedOverride , bool $inline )
 {
     global $entities;
     global $expected;
@@ -154,28 +163,56 @@ function push( string $path , string $contents , bool $expectedOverride )
     else
         $override[$name]++;
 
-    $entities[$name] = $contents;
+    $entity = new EntityData( $path , $name , $text , $inline );
+    $entities[$name] = $entity;
 }
 
-function dump( array $entities )
+function dump( string $filename , array $entities )
 {
-    global $generate;
-
     // In PHP 8.4 may be possible to construct an extended
     // DOMEntity class with writable properties. For now,
     // creating entities files directly as text.
 
-    $file = fopen( $generate , "w" );
+    $file = fopen( $filename , "w" );
     fputs( $file , "\n<!-- DO NOT COPY - Autogenerated by entities.php -->\n\n" );
 
-    foreach( $entities as $name => $text )
+    foreach( $entities as $name => $entity )
     {
-        $text = str_replace( "'" , '&apos;' , $text );
-        fputs( $file , "<!ENTITY $name '$text'>\n\n");
+        if ( $entity->inline )
+        {
+            $text = str_replace( "'" , '&apos;' , $entity->text );
+            fputs( $file , "<!ENTITY $name '$text'>\n\n");
+        }
+        else
+        {
+            fputs( $file , "<!ENTITY $name SYSTEM '{$entity->path}'>\n\n");
+        }
     }
     fclose( $file );
 }
 
+function shouldInline( DOMDocument $dom ) : bool
+{
+    // Pure text entities CANNOT be SYSTEMed (or libxml fails).
+    // But entities that CONTAINS elements NEED to be SYSTEMed
+    // to avoid quotation madness.
+
+    // Why libxml/w3c? WHY?
+
+    $xpath = new DomXPath( $dom );
+    $elems = $xpath->query( "child::*" );
+    return ( $elems->length == 0 );
+}
+
+function shouldInlineRecurse( DOMNode $node ) : bool
+{
+    if ( $node->nodeType == XML_ELEMENT_NODE )
+        return true;
+    foreach ( $node->childNodes as $node )
+        return shouldInlineRecurse( $node );
+
+}
+
 function verifyOverrides( bool $outputDetail )
 {
     global $entities;
@@ -211,4 +248,4 @@ function verifyOverrides( bool $outputDetail )
     }
 
     return [$countGenerated, $countExpectedOverriden, $countUnexpectedOverriden];
-}
+}