Add an SSG module for interacting with SSG (#12)

Policies and Rules come from the SCAP Security Guide, generated from
https://github.com/ComplianceAsCode/content. This adds rake tasks to
download and unarchive released versions of SSG for a given application,
and a shortcut rake task for all the RHEL SSG content.

Signed-off-by: Andrew Kofink <akofink@redhat.com>

Author: akofink

README.md

@@ -35,6 +35,17 @@ parser.rule_results # [#<OpenscapParser::RuleResult:0x00005576e8022f60 @id="xccd
 # and more!
 ```
 
+### Fetching SCAP Security Guide Content
+
+This gem includes a rake task to sync content from the [ComplianceAsCode project](https://github.com/ComplianceAsCode/content). The following examples show how to download and exract datastream files from the released versions:
+
+```sh
+rake ssg:sync DATASTREAMS=latest:fedora # fetch and extract the latest fedora datastream
+rake ssg:sync DATASTREAMS=v0.1.45:fedora,v0.1.45:firefox # fetch and extract tag v0.1.45 for fedora and firefox datastreams
+rake ssg:sync_rhel # fetch and extract the latest released versions of the RHEL 6, 7, and 8 datastreams
+```
+
+An SSG version will be downloaded only once, even if it is specified multiple times for multiple datastreams.
 
 ## Development
 

Rakefile

@@ -1,6 +1,8 @@
 require "bundler/gem_tasks"
 require "rake/testtask"
 
+import "./lib/tasks/ssg.rake"
+
 Rake::TestTask.new(:test) do |t|
   t.libs << "test"
   t.libs << "lib"

lib/ssg.rb

@@ -0,0 +1,5 @@
+require 'ssg/downloader'
+require 'ssg/unarchiver'
+
+module Ssg
+end

lib/ssg/downloader.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'net/http'
+
+module Ssg
+  # Downloads SCAP datastreams from the SCAP Security Guide
+  # https://github.com/ComplianceAsCode/content
+  class Downloader
+    RELEASES_API = 'https://api.github.com/repos'\
+                             '/ComplianceAsCode/content/releases/'
+    SSG_DS_REGEX = /scap-security-guide-(\d+\.)+zip$/
+
+    def initialize(version = 'latest')
+      @release_uri = URI(
+        "#{RELEASES_API}#{'tags/' unless version[/^latest$/]}#{version}"
+      )
+    end
+
+    def self.download!(versions = [])
+      versions.uniq.map do |version|
+        [version, new(version).fetch_datastream_file]
+      end.to_h
+    end
+
+    def fetch_datastream_file
+      puts "Fetching #{datastream_filename}"
+      get_chunked(datastream_uri)
+
+      datastream_filename
+    end
+
+    private
+
+    def datastream_uri
+      @datastream_uri ||= URI(
+        download_urls.find { |url| url[SSG_DS_REGEX] }
+      )
+    end
+
+    def download_urls
+      get_json(@release_uri).dig('assets').map do |asset|
+        asset.dig('browser_download_url')
+      end
+    end
+
+    def fetch(request, &block)
+      Net::HTTP.start(
+        request.uri.host, request.uri.port,
+        use_ssl: request.uri.scheme['https']
+      ) do |http|
+        check_response(http.request(request, &block), &block)
+      end
+    end
+
+    def get(uri, &block)
+      fetch(Net::HTTP::Get.new(uri), &block)
+    end
+
+    def head(uri, &block)
+      fetch(Net::HTTP::Head.new(uri), &block)
+    end
+
+    def check_response(response, &block)
+      case response
+      when Net::HTTPSuccess
+        response
+      when Net::HTTPRedirection
+        get(URI(response['location']), &block)
+      else
+        response.value
+      end
+    end
+
+    def get_chunked(uri, filename: datastream_filename)
+      head(uri) do |response|
+        next unless Net::HTTPSuccess === response
+        open(filename, 'wb') do |file|
+          response.read_body do |chunk|
+            file.write(chunk)
+          end
+        end
+      end
+    end
+
+    def datastream_filename
+      datastream_uri.path.split('/').last[SSG_DS_REGEX]
+    end
+
+    def get_json(uri)
+      JSON.parse(get(uri).body)
+    end
+  end
+end

lib/ssg/unarchiver.rb

@@ -0,0 +1,34 @@
+module Ssg
+  class Unarchiver
+    UNZIP_CMD = ['unzip', '-o']
+
+    def initialize(ds_zip_filename, datastreams)
+      @ds_zip_filename = ds_zip_filename
+      @datastreams = datastreams
+    end
+
+    def self.unarchive!(ds_zip_filenames, datastreams)
+      ds_zip_filenames.map do |version, ds_zip_filename|
+        new(ds_zip_filename, [datastreams[version]].flatten).datastream_files
+      end
+    end
+
+    def datastream_files
+      datastream_filenames if system(
+        *UNZIP_CMD, @ds_zip_filename, *datastream_filenames
+      )
+    end
+
+    private
+
+    def datastream_filenames
+      @datastreams.map do |datastream|
+        "#{datastream_dir}/ssg-#{datastream}-ds.xml"
+      end
+    end
+
+    def datastream_dir
+      @ds_zip_filename.split('.')[0...-1].join('.')
+    end
+  end
+end

lib/tasks/ssg.rake

@@ -0,0 +1,32 @@
+desc 'Import or update SCAP datastreams from the SCAP Security Guide'
+namespace :ssg do
+  desc 'Import or update SCAP datastreams for RHEL 6, 7, and 8'
+  task :sync_rhel do |task|
+    RHEL_SSG_VERSIONS = (
+      'v0.1.28:rhel6,'\
+      'v0.1.43:rhel7,'\
+      'v0.1.42:rhel8'
+    )
+
+    ENV['DATASTREAMS'] = RHEL_SSG_VERSIONS
+    Rake::Task['ssg:sync'].invoke
+  end
+
+  desc 'Import or update SCAP datastreams, '\
+    'provided as a comma separated list: '\
+    '`rake ssg:sync DATASTREAMS=v0.1.43:rhel7,latest:fedora`'
+  task :sync do |task|
+    DATASTREAMS = ENV.fetch('DATASTREAMS', '').split(',')
+      .inject({}) do |datastreams, arg|
+      version, datastream = arg.split(':')
+      datastreams[version] = (datastreams[version] || []).push(datastream)
+
+      datastreams
+    end
+
+    require 'ssg'
+
+    ds_zip_filenames = Ssg::Downloader.download!(DATASTREAMS.keys)
+    Ssg::Unarchiver.unarchive!(ds_zip_filenames, DATASTREAMS)
+  end
+end

openscap_parser.gemspec

@@ -40,6 +40,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "mocha", "~> 1.0"
   spec.add_development_dependency "shoulda-context"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "pry-byebug"

test/ssg/downloader_test.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+require 'ssg/downloader'
+
+module Ssg
+  class DownloaderTest < MiniTest::Test
+    context 'fetch_datastream_file' do
+      test 'returns the fetched file' do
+        FILE = 'scap-security-guide-0.0.0.zip'
+        uri = URI("https://example.com/#{FILE}")
+        downloader = Downloader.new
+        downloader.expects(:datastream_uri).
+          at_least_once.returns(uri)
+        downloader.expects(:get_chunked).with uri
+
+        assert_equal FILE, downloader.fetch_datastream_file
+      end
+    end
+  end
+end

test/ssg/unarchiver_test.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+require 'ssg/unarchiver'
+
+module Ssg
+  class UnarchiverTest < MiniTest::Test
+    context 'datastream_files' do
+      test 'properly shells out to unzip' do
+        ZIP_FILE = 'scap-security-guide-0.0.0.zip'
+        DATASTREAMS = ['rhel6']
+        FILES = []
+        unarchiver = Unarchiver.new(ZIP_FILE, DATASTREAMS)
+        unarchiver.expects(:system).with(
+          "unzip", "-o",
+          "scap-security-guide-0.0.0.zip",
+          "scap-security-guide-0.0.0/ssg-rhel6-ds.xml"
+        ).returns(true)
+
+        assert_equal ['scap-security-guide-0.0.0/ssg-rhel6-ds.xml'],
+          unarchiver.datastream_files
+      end
+    end
+  end
+end

test/test_helper.rb

@@ -4,6 +4,7 @@
 
 require "minitest/autorun"
 require 'shoulda-context'
+require 'mocha/minitest'
 
 def test(name, &block)
   test_name = "test_#{name.gsub(/\s+/, '_')}".to_sym