Robots: Use admin_url() for the robots.txt admin paths.

Derive the `Disallow` and `Allow` directives in `do_robots()` from `admin_url()` and `admin_url( 'admin-ajax.php' )` rather than from a hardcoded `/wp-admin/` path built off `site_url()`. Only the URL path portion is emitted, as before, so installs that relocate or filter their admin URL now produce a correct default `robots.txt`.

Also guard the `Content-Type` header with a `headers_sent()` check to avoid a warning when the headers have already been sent.

Developed in WordPress#11998.
Follow-up to r34985.

Props masteradhoc, yogeshbhutkar, hrohh, westonruter, mukeshpanchal27, 1ucay.
Fixes #63467.


git-svn-id: https://develop.svn.wordpress.org/trunk@62633 602fd350-edb4-49c9-b593-d223f7449a82

Author: westonruter

src/wp-includes/functions.php

@@ -1711,7 +1711,9 @@ function do_feed_atom( $for_comments ) {
  *              filter callback.
  */
 function do_robots() {
-	header( 'Content-Type: text/plain; charset=utf-8' );
+	if ( ! headers_sent() ) {
+		header( 'Content-Type: text/plain; charset=utf-8' );
+	}
 
 	/**
 	 * Fires when displaying the robots.txt file.
@@ -1723,10 +1725,8 @@ function do_robots() {
 	$output = "User-agent: *\n";
 	$public = (bool) get_option( 'blog_public' );
 
-	$site_url = parse_url( site_url() );
-	$path     = ( ! empty( $site_url['path'] ) ) ? $site_url['path'] : '';
-	$output  .= "Disallow: $path/wp-admin/\n";
-	$output  .= "Allow: $path/wp-admin/admin-ajax.php\n";
+	$output .= 'Disallow: ' . wp_parse_url( admin_url(), PHP_URL_PATH ) . "\n";
+	$output .= 'Allow: ' . wp_parse_url( admin_url( 'admin-ajax.php' ), PHP_URL_PATH ) . "\n";
 
 	/**
 	 * Filters the robots.txt output.

tests/phpunit/tests/robots.php

@@ -144,6 +144,26 @@ public function test_wp_robots_non_search_page() {
 		$this->assertStringNotContainsString( 'noindex', $output );
 	}
 
+	/**
+	 * @ticket 63467
+	 */
+	public function test_do_robots_uses_filtered_admin_url_paths(): void {
+		add_filter(
+			'admin_url',
+			static function ( string $url, string $path, ?int $blog_id, string $scheme ): string {
+				return home_url( "/control/$path", $scheme );
+			},
+			10,
+			4
+		);
+
+		$output = get_echo( 'do_robots' );
+
+		$this->assertStringNotContainsString( 'wp-admin', $output );
+		$this->assertStringContainsString( "Disallow: /control/\n", $output );
+		$this->assertStringContainsString( "Allow: /control/admin-ajax.php\n", $output );
+	}
+
 	public function add_noindex_directive( array $robots ) {
 		$robots['noindex'] = true;
 		return $robots;