diff --git a/configure.php b/configure.php index 1a07591639..85c09b77cd 100755 --- a/configure.php +++ b/configure.php @@ -652,11 +652,43 @@ function dtd_text_entities() } checkvalue($ac["GENERATE"]); -function dom_load( DOMDocument $dom , string $filename , string $baseURI = "" ) : bool +echo "Creating monolithic temp/manual.xml... "; +$dom = new DOMDocument(); + +if ( dom_load( $dom , "{$ac['srcdir']}/{$ac["INPUT_FILENAME"]}" ) ) +{ + dom_saveload( $dom ); // correct file/line/column on error messages + echo " done.\n"; +} +else +{ + echo "failed.\n"; + print_xml_errors(); + xml_broken_file_check(); + errors_are_bad(1); +} + +function dom_load( DOMDocument $dom , string $filename , bool $firstLoad = true ) : bool { $filename = realpath( $filename ); - $options = LIBXML_NOENT | LIBXML_COMPACT | LIBXML_BIGLINES | LIBXML_PARSEHUGE; - return $dom->load( $filename , $options ); + + // On the first load we cannot use LIBXML_NSCLEAN, because + // libxml drops all namespaces inside DTD entities. + + $options = LIBXML_NOENT + | LIBXML_COMPACT + | LIBXML_BIGLINES + | LIBXML_PARSEHUGE; + + if ( ! $firstLoad ) + $options |= LIBXML_NSCLEAN; + + $ret = $dom->load( $filename , $options ); + + if ( $ret && $firstLoad ) + xml_trim_first( $dom ); + + return $ret; } function dom_saveload( DOMDocument $dom , string $filename = "" ) : string @@ -666,28 +698,88 @@ function dom_saveload( DOMDocument $dom , string $filename = "" ) : string libxml_clear_errors(); $dom->save( $filename ); - dom_load( $dom , $filename ); + dom_load( $dom , $filename , false ); return $filename; } -echo "Creating monolithic temp/manual.xml... "; -$dom = new DOMDocument(); - -if ( dom_load( $dom , "{$ac['srcdir']}/{$ac["INPUT_FILENAME"]}" ) ) +function xml_trim_first( DOMDocument $doc ) { - dom_saveload( $dom ); // correct file/line/column on error messages - echo " done.\n"; + $xpath = new DOMXPath( $doc ); + $dtdNode = null; + $dels = []; + + // Remove DTD Document Type, as all entity references + // are already expanded at this point. + + foreach( $doc->childNodes as $node ) + if ( $node->nodeType == XML_DOCUMENT_TYPE_NODE ) + $dtdNode = $node; + if ( $dtdNode != null ) + $node->parentNode->removeChild( $dtdNode ); + + // Remove all XML comments, as they are distracting, + // in reverse order, outside enumerations. + + $comments = $xpath->query( "//comment()" ); + for ( $idx = $comments->length - 1 ; $idx >= 0 ; $idx-- ) + { + $node = $comments[ $idx ]; + $node->parentNode->removeChild( $node ); + } + + // See docbook/docbookwsi.php. + + // These first five elements account for almost 50% + // of all insignificant Docbook whitespace in manual. + + $trimList[] = 'refsect1'; + $trimList[] = 'varlistentry'; + $trimList[] = 'listitem'; + $trimList[] = 'row'; + $trimList[] = 'methodsynopsis'; + + // The first twenty elements account for almost 95% + // of all insignificant Docbook whitespace in manual. + + $trimList[] = 'refentry'; + $trimList[] = 'fieldsynopsis'; + $trimList[] = 'variablelist'; + $trimList[] = 'example'; + $trimList[] = 'simplelist'; + $trimList[] = 'refnamediv'; + $trimList[] = 'tbody'; + $trimList[] = 'reference'; + $trimList[] = 'classsynopsis'; + $trimList[] = 'section'; + $trimList[] = 'tgroup'; + $trimList[] = 'thead'; + $trimList[] = 'itemizedlist'; + $trimList[] = 'note'; + $trimList[] = 'sect2'; + + xml_trim_docbook_wsi( $doc->documentElement , $trimList ); } -else + +function xml_trim_docbook_wsi( DOMNode $parent , array $trimList ) { - echo "failed.\n"; - print_xml_errors(); - individual_xml_broken_check(); - errors_are_bad(1); + $nodeList = $parent->childNodes; + for ( $idx = $nodeList->length - 1 ; $idx >= 0 ; $idx-- ) + { + $node = $nodeList[ $idx ]; + $type = $node->nodeType; + + if ( $type == 1 ) // XML_ELEMENT_NODE + xml_trim_docbook_wsi( $node , $trimList ); + + if ( $type == 3 ) // XML_TEXT_NODE + if ( in_array( $parent->nodeName , $trimList ) ) + if ( trim( $node->nodeValue ) == '' ) + $parent->removeChild( $node ); + } } -function individual_xml_broken_check() +function xml_broken_file_check() { $cmd = array(); $cmd[] = $GLOBALS['ac']['PHP']; diff --git a/docbook/docbookwsi.php b/docbook/docbookwsi.php new file mode 100644 index 0000000000..58a20ef002 --- /dev/null +++ b/docbook/docbookwsi.php @@ -0,0 +1,193 @@ + +/* +This script reads a RelaxNG XML file, and calculates all elements +that _not_ contain contents, in all alternatives. That is, +the list shows all elements that can have all inter-element +whitespace removed, without affecting the expected parsing of the +XML document that follows the RelaxNG specification. + +If run with a second XML argument, the script will calculate all savings +that can be done by stripping these insignificant whitespace between +elements. + +See mentions of 'docbookwsi' on source code for parts that need to be +updated if/when a new version of Docbook is used. */ + +$arg0 = array_shift( $argv ) ?? null; +$rngFile = array_shift( $argv ) ?? null; +$xmlFile = array_shift( $argv ) ?? null; + +if ( $rngFile == null ) +{ + print "Usage: '$argv0' rngFile [xmlFile]\n\n"; + return; +} + +$list = generate_element_trim_list( $rngFile ); + +if ( $xmlFile == null ) +{ + foreach( $list as $elem => $hasText ) + if ( ! $hasText ) + print "$elem\n"; + exit( 0 ); +} +else + xml_trim_stats( $xmlFile , $list ); + +exit( 0 ); + +function generate_element_trim_list( string $rngFilename ) : array +{ + $doc = new DOMDocument(); + if ( ! $doc->load( $rngFilename , LIBXML_NOBLANKS ) ) + throw new Exception( "XML load failed.\n" ); + + // First, we get all elements definitions that directly + // mentions , and also gather all s they refer. + + $elemText = []; + $elemRefs = []; + + $xpath1 = new DOMXpath( $doc ); + $xpath2 = new DOMXpath( $doc ); + $xpath1->registerNamespace ( 'rng' , 'http://relaxng.org/ns/structure/1.0' ); + $xpath2->registerNamespace ( 'rng' , 'http://relaxng.org/ns/structure/1.0' ); + + $list = $xpath1->query( '//rng:element' ); + foreach( $list as $elem ) + { + $name = $elem->getAttribute( 'name' ); + if ( $name == '' ) + continue; + + $text = count ( $xpath2->query( './/rng:text' , $elem ) ); + $refs = $xpath2->query( './/rng:ref' , $elem ); + + $elemText[ $name ] = $text; + $elemRefs[ $name ] = []; + + foreach( $refs as $ref ) + { + $refName = $ref->getAttribute( 'name' ); + $elemRefs[ $name ][] = $refName; + } + } + + unset( $xpath1 ); + unset( $xpath2 ); + + // After all elements are collected, and directly textual elements + // are marked, we can remove all s, as they cannot influence + // if a parent element is trimmable or not, and so that any + // inside of a cannot be found by XPaths, while exploring + // the original 's erences. + + $xpath3 = new DOMXpath( $doc ); + $xpath3->registerNamespace ( 'rng' , 'http://relaxng.org/ns/structure/1.0' ); + + $todoDels = []; + $dels = $xpath3->query( '//rng:element' ); + foreach( $dels as $del ) + array_push( $todoDels , $del ); + foreach( $todoDels as $del ) + $del->parentNode->removeChild( $del ); + + // Then, we explore all references of all elements, for + // indirect mentions of s. + + foreach( $elemText as $name => $text ) + { + $text = element_references_contains_text( $doc , $name , $elemRefs[ $name ] ); + $elemText[ $name ] |= $text; + } + + return $elemText; +} + +function element_references_contains_text( DOMDocument $doc , string $elemName , array $refs ) : bool +{ + $ret = false; + $doneRefs = []; + $todoRefs = array_unique( $refs ); + + $xpath = new DOMXpath( $doc ); + $xpath->registerNamespace ( 'rng' , 'http://relaxng.org/ns/structure/1.0' ); + + while ( ( $refName = array_pop( $todoRefs ) ) != null ) + { + $doneRefs[ $refName ] = true; + + $defs = $xpath->query( "//rng:define[@name='$refName']" ); + if ( $defs->count() != 1 ) + throw new Exception( "Unique define search failed for '$refName'." ); + $def = $defs[0]; + + $text = count ( $xpath->query( './/rng:text' , $def ) ); + if ( $text ) + return true; + + $subRefs = $xpath->query( './/rng:ref' , $def ); + foreach( $subRefs as $subRef ) + { + $subRefName = $subRef->getAttribute( 'name' ); + if ( isset( $doneRefs[ $subRefName ] ) ) + continue; + $todoRefs[] = $subRefName; + } + } + + return false; +} + +function xml_trim_stats( string $xmlFilename , array $elemText ) +{ + $doc = new DOMDocument(); + if ( ! $doc->load( $xmlFilename ) ) + throw new Exception( "XML load failed.\n" ); + + $stats = []; + xml_trim_stats_enter( $doc->documentElement , $elemText, $stats ); + arsort( $stats ); + + $total = 0; + foreach( $stats as $elem => $trimSize ) + { + print "$trimSize $elem\n"; + $total += $trimSize; + } + print "\ntotal $total\n"; +} + +function xml_trim_stats_enter( DOMNode $node , array $elemText , array & $stats , int $level = 0 ) +{ + $name = $node->nodeName; + $text = $elemText[ $name ] ?? true; + + if ( ! $text ) + { + $size = 0; + $dels = []; + + foreach( $node->childNodes as $child ) + if ( $child->nodeType == XML_TEXT_NODE ) + if ( trim( $child->nodeValue ) == '' ) + $dels[] = $child; + + foreach( $dels as $del ) + { + $size += strlen( $del->nodeValue ); + $del->parentNode->removeChild( $del ); + } + + if ( isset( $stats[ $name ] ) ) + $stats[ $name ] += $size; + else + $stats[ $name ] = $size; + } + + foreach( $node->childNodes as $child ) + xml_trim_stats_enter( $child , $elemText , $stats , $level + 1 ); +} \ No newline at end of file