Refactoring

Author: JanPetterMG

.gitignore

@@ -1,4 +1,3 @@
 composer.lock
 composer.phar
 /vendor/
-/build/logs/*

.travis.yml

@@ -1,14 +1,13 @@
 sudo: false
 language: php
 php:
+  - 7.1
   - 7.0
   - 5.6
   - hhvm
 matrix:
   allow_failures:
     - php: hhvm
-before_install:
-  - composer selfupdate
 install:
   - composer install
 after_script:

README.md

@@ -1,10 +1,10 @@
-[![Build Status](https://travis-ci.org/VIPnytt/X-Robots-Tag-parser.svg?branch=master)](https://travis-ci.org/VIPnytt/X-Robots-Tag-parser)
-[![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/VIPnytt/X-Robots-Tag-parser/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/VIPnytt/X-Robots-Tag-parser/?branch=master)
-[![Code Climate](https://codeclimate.com/github/VIPnytt/X-Robots-Tag-parser/badges/gpa.svg)](https://codeclimate.com/github/VIPnytt/X-Robots-Tag-parser)
-[![Test Coverage](https://codeclimate.com/github/VIPnytt/X-Robots-Tag-parser/badges/coverage.svg)](https://codeclimate.com/github/VIPnytt/X-Robots-Tag-parser/coverage)
-[![License](https://poser.pugx.org/VIPnytt/X-Robots-Tag-parser/license)](https://github.com/VIPnytt/X-Robots-Tag-parser/blob/master/LICENSE)
-[![Packagist](https://img.shields.io/packagist/v/vipnytt/x-robots-tag-parser.svg)](https://packagist.org/packages/vipnytt/x-robots-tag-parser)
-[![Join the chat at https://gitter.im/VIPnytt/X-Robots-Tag-parser](https://badges.gitter.im/VIPnytt/X-Robots-Tag-parser.svg)](https://gitter.im/VIPnytt/X-Robots-Tag-parser)
+[![Build Status](https://travis-ci.org/VIPnytt/RobotsTagParser.svg?branch=master)](https://travis-ci.org/VIPnytt/RobotsTagParser)
+[![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/VIPnytt/RobotsTagParser/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/VIPnytt/RobotsTagParser/?branch=master)
+[![Code Climate](https://codeclimate.com/github/VIPnytt/RobotsTagParser/badges/gpa.svg)](https://codeclimate.com/github/VIPnytt/RobotsTagParser)
+[![Test Coverage](https://codeclimate.com/github/VIPnytt/RobotsTagParser/badges/coverage.svg)](https://codeclimate.com/github/VIPnytt/RobotsTagParser/coverage)
+[![License](https://poser.pugx.org/VIPnytt/RobotsTagParser/license)](https://github.com/VIPnytt/RobotsTagParser/blob/master/LICENSE)
+[![Packagist](https://img.shields.io/packagist/v/vipnytt/robotstagparser.svg)](https://packagist.org/packages/vipnytt/robotstagparser)
+[![Gitter](https://badges.gitter.im/VIPnytt/RobotsTagParser.svg)](https://gitter.im/VIPnytt/RobotsTagParser)
 
 # X-Robots-Tag HTTP header parser
 PHP class to parse X-Robots-Tag HTTP headers according to [Google X-Robots-Tag HTTP header specifications](https://developers.google.com/webmasters/control-crawl-index/docs/robots_meta_tag#using-the-x-robots-tag-http-header).
@@ -23,7 +23,7 @@ The library is available via [Composer](https://getcomposer.org). Add this to yo
 ```json
 {
     "require": {
-        "vipnytt/x-robots-tag-parser": "~0.2"
+        "vipnytt/robotstagparser": "~0.3"
     }
 }
 ```
@@ -94,16 +94,16 @@ $parser = new XRobotsTagParser('myUserAgent', $headers);
 $array = $parser->export();
 ```
 
-## Supported directives:
-- [x] ````all```` - There are no restrictions for indexing or serving.
-- [x] ````none```` - Equivalent to ````noindex```` and ````nofollow````.
-- [x] ````noindex```` - Do not show this page in search results and do not show a "Cached" link in search results.
-- [x] ````nofollow```` - Do not follow the links on this page.
-- [x] ````noarchive```` - Do not show a "Cached" link in search results.
-- [x] ````nosnippet```` - Do not show a snippet in the search results for this page.
-- [x] ````noodp```` - Do not use metadata from the [Open Directory project](http://dmoz.org/) for titles or snippets shown for this page.
-- [x] ````notranslate```` - Do not offer translation of this page in search results.
-- [x] ````noimageindex```` - Do not index images on this page.
-- [x] ````unavailable_after```` - Do not show this page in search results after the specified date/time.
+## Directives:
+- [x] `all` - There are no restrictions for indexing or serving.
+- [x] `none` - Equivalent to `noindex` and `nofollow`.
+- [x] `noindex` - Do not show this page in search results and do not show a "Cached" link in search results.
+- [x] `nofollow` - Do not follow the links on this page.
+- [x] `noarchive` - Do not show a "Cached" link in search results.
+- [x] `nosnippet` - Do not show a snippet in the search results for this page.
+- [x] `noodp` - Do not use metadata from the [Open Directory project](http://dmoz.org/) for titles or snippets shown for this page.
+- [x] `notranslate` - Do not offer translation of this page in search results.
+- [x] `noimageindex` - Do not index images on this page.
+- [x] `unavailable_after` - Do not show this page in search results after the specified date/time.
 
 Source: [https://developers.google.com/webmasters/control-crawl-index/docs/robots_meta_tag](https://developers.google.com/webmasters/control-crawl-index/docs/robots_meta_tag#valid-indexing--serving-directives)

build/.gitkeep

@@ -1,5 +1,5 @@
 {
-  "name": "vipnytt/x-robots-tag-parser",
+  "name": "vipnytt/robotstagparser",
   "description": "X-Robots-Tag HTTP header parser class",
   "type": "library",
   "keywords": [
@@ -13,7 +13,7 @@
     "Spider",
     "Web-Crawler"
   ],
-  "homepage": "https://github.com/VIPnytt/X-Robots-Tag-parser",
+  "homepage": "https://github.com/VIPnytt/RobotsTagParser",
   "license": "MIT",
   "authors": [
     {
@@ -29,21 +29,24 @@
   ],
   "support": {
     "email": "vipnytt@gmail.com",
-    "issues": "https://github.com/VIPnytt/X-Robots-Tag-parser/issues",
-    "wiki": "https://github.com/VIPnytt/X-Robots-Tag-parser/wiki",
-    "source": "https://github.com/VIPnytt/X-Robots-Tag-parser",
-    "docs": "https://github.com/VIPnytt/X-Robots-Tag-parser/wiki"
+    "issues": "https://github.com/VIPnytt/RobotsTagParser/issues",
+    "wiki": "https://github.com/VIPnytt/RobotsTagParser/wiki",
+    "source": "https://github.com/VIPnytt/RobotsTagParser",
+    "docs": "https://github.com/VIPnytt/RobotsTagParser/wiki"
   },
   "require": {
     "php": ">=5.6",
     "guzzlehttp/guzzle": "6.*",
-    "vipnytt/useragentparser": "~0.2",
+    "vipnytt/useragentparser": "~1.0",
     "ext-mbstring": "*"
   },
   "require-dev": {
     "phpunit/phpunit": ">=4.0",
     "codeclimate/php-test-reporter": ">=0.2.0"
   },
+  "suggest": {
+    "vipnytt/robotstxtparser": "Robots.txt parser."
+  },
   "autoload": {
     "psr-4": {
       "vipnytt\\": "src/"

build/logs/.gitkeep

@@ -3,7 +3,6 @@
 
 use vipnytt\XRobotsTagParser\Directives;
 use vipnytt\XRobotsTagParser\Exceptions\XRobotsTagParserException;
-use vipnytt\XRobotsTagParser\Rebuild;
 use vipnytt\XRobotsTagParser\RobotsTagInterface;
 
 /**
@@ -16,7 +15,7 @@
  * @author Jan-Petter Gundersen (europe.jpg@gmail.com)
  *
  * Project:
- * @link https://github.com/VIPnytt/X-Robots-Tag-parser
+ * @link https://github.com/VIPnytt/RobotsTagParser
  * @license https://opensource.org/licenses/MIT MIT license
  *
  * Specification:
@@ -29,14 +28,14 @@ class XRobotsTagParser implements RobotsTagInterface
      *
      * @var string
      */
-    protected $userAgent = '';
+    protected $userAgent = self::USER_AGENT;
 
     /**
      * User-Agent for rule selection
      *
      * @var string
      */
-    protected $userAgentMatch = '';
+    protected $userAgentMatch = self::USER_AGENT;
 
     /**
      * Current rule
@@ -63,12 +62,12 @@ class XRobotsTagParser implements RobotsTagInterface
      * Constructor
      *
      * @param string $userAgent
-     * @param array $headers
+     * @param array|null $headers
      */
-    public function __construct($userAgent = '', $headers = null)
+    public function __construct($userAgent = self::USER_AGENT, $headers = null)
     {
         $this->userAgent = $userAgent;
-        if (isset($headers)) {
+        if (!empty($headers)) {
             $this->parse($headers);
         }
     }
@@ -102,13 +101,13 @@ protected function detectDirectives()
     {
         $directives = array_map('trim', mb_split(',', $this->currentRule));
         $pair = array_map('trim', mb_split(':', $directives[0], 2));
-        if (count($pair) == 2 && !in_array($pair[0], array_keys(self::DIRECTIVES))) {
+        if (count($pair) == 2 && !in_array($pair[0], self::DIRECTIVES)) {
             $this->currentUserAgent = $pair[0];
             $directives[0] = $pair[1];
         }
         foreach ($directives as $rule) {
             $directive = trim(mb_split(':', $rule, 2)[0]);
-            if (in_array($directive, array_keys(self::DIRECTIVES))) {
+            if (in_array($directive, self::DIRECTIVES)) {
                 $this->addRule($directive);
             }
         }
@@ -145,7 +144,7 @@ protected function addRule($directive)
     protected function cleanup()
     {
         $this->currentRule = '';
-        $this->currentUserAgent = '';
+        $this->currentUserAgent = self::USER_AGENT;
     }
 
     /**
@@ -156,32 +155,26 @@ protected function cleanup()
     protected function matchUserAgent()
     {
         $userAgentParser = new UserAgentParser($this->userAgent);
-        $match = $userAgentParser->match(array_keys($this->rules));
-        $this->userAgentMatch = ($match !== false) ? $match : '';
+        $this->userAgentMatch = (($match = $userAgentParser->getMostSpecific(array_keys($this->rules))) !== false) ? $match : self::USER_AGENT;
         return $this->userAgentMatch;
     }
 
     /**
      * Return all applicable rules
      *
-     * @param bool $raw
      * @return array
      */
-    public function getRules($raw = false)
+    public function getRules()
     {
         $rules = [];
         // Default UserAgent
-        if (isset($this->rules[''])) {
-            $rules = array_merge($rules, $this->rules['']);
+        if (isset($this->rules[self::USER_AGENT])) {
+            $rules = array_merge($rules, $this->rules[self::USER_AGENT]);
         }
         // Matching UserAgent
         if (isset($this->rules[$this->userAgentMatch])) {
             $rules = array_merge($rules, $this->rules[$this->userAgentMatch]);
         }
-        if (!$raw) {
-            $rebuild = new Rebuild($rules);
-            $rules = $rebuild->getResult();
-        }
         // Result
         return $rules;
     }
@@ -195,20 +188,4 @@ public function export()
     {
         return $this->rules;
     }
-
-    /**
-     * Get the meaning of an Directive
-     *
-     * @param string $directive
-     * @return string
-     * @throws XRobotsTagParserException
-     */
-    public function getDirectiveMeaning($directive)
-    {
-        $directive = mb_strtolower($directive);
-        if (!in_array($directive, array_keys(self::DIRECTIVES))) {
-            throw new XRobotsTagParserException('Unknown directive');
-        }
-        return self::DIRECTIVES[$directive];
-    }
 }

composer.json

@@ -20,7 +20,7 @@ class TextString extends XRobotsTagParser
      */
     public function __construct($string, $userAgent = '')
     {
-        $array = array_map('trim', mb_split('\r\n|\n|\r', $string));
+        $array = array_map('trim', mb_split('\r\n|\r|\n', $string));
         parent::__construct($userAgent, $array);
     }
 }

src/XRobotsTagParser.php

@@ -18,11 +18,18 @@ class Url extends XRobotsTagParser\Adapters\GuzzleHttp
      * @var array
      */
     protected $config = [
+        'allow_redirects' => [
+            'referer' => true,
+            'strict' => false,
+        ],
+        'connect_timeout' => 30,
+        'decode_content' => true,
         'headers' => [
-            'User-Agent' => 'X-Robots-Tag-parser-VIPnytt/1.0 (+https://github.com/VIPnytt/X-Robots-Tag-parser/blob/master/README.md)',
+            'user-agent' => 'XRobotsTagParser-VIPnytt/1.0 (+https://github.com/VIPnytt/RobotsTagParser/blob/master/README.md)',
         ],
         'http_errors' => false,
-        'timeout' => 30,
+        'timeout' => 120,
+        'verify' => true,
     ];
 
     /**

src/XRobotsTagParser/Adapters/TextString.php

@@ -1,11 +1,6 @@
 <?php
 namespace vipnytt\XRobotsTagParser\Directives;
 
-/**
- * Interface DirectiveInterface
- *
- * @package vipnytt\XRobotsTagParser\Directives
- */
 interface DirectiveInterface
 {
     /**