replace faraday by http.rb

 add  demo for thread pool
 add benchmark class
 add more comment partially generated by CodeGemma running locally

Author: jvmvik

Gemfile

@@ -3,12 +3,10 @@ source 'https://rubygems.org'
 gemspec
 
 group :development, :production do
-  gem 'faraday', '~> 2.13'
+  gem 'http', '~> 0.13.3'
 end
 
 group :test, :development do
-  # HTTP client implementation
-  gem 'faraday-httpclient'
   # code coloring for yard
   gem 'redcarpet'
   # documentation generation
@@ -20,12 +18,11 @@ group :test, :development do
   # code coverage to monitor rspec tests
   gem 'simplecov'
 
-  # for client_async_spec.rb
-  gem 'async-http-faraday'
+  # Thread pool example file: thread_pool_spec.rb
+  # https://github.com/mperham/connection_pool/tree/main
+  gem 'connection_pool'
+
+  # Benchmark is no longer included in Ruby 3.5
+  gem 'benchmark'
 
-  # platforms :mri do
-  #   gem "byebug"
-  #   gem "pry"
-  #   gem "pry-byebug"
-  # end
 end

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2018-2023 SerpApi
+Copyright (c) 2018-2025 SerpApi
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -160,47 +160,6 @@ pp client.account
 
 It prints your account information.
 
-### Change HTTP client implementation
-
-The internal HTTP client is based on the [Faraday](https://lostisland.github.io/faraday/) library.
- You can change the HTTP client by passing the adapter as an argument to the `SerpApi::Client.new` method.
-
-It requires to install the adapter gem.
-```sh
-gem install faraday-httpclient
-```
-
-```ruby
-# see faraday documentation for adapter description.
-client = SerpApi::Client.new(engine: 'google', api_key: ENV['API_KEY'], timeout: 10, adapter: :httpclient)
-data = client.search(q: 'Coffee', location: 'Austin, TX')
-pp data
-```
-
-or a more [advanced parallel requests example](https://lostisland.github.io/faraday/#/advanced/parallel-requests)
-
-```ruby
-# install and load
-require 'async/http/faraday'
-
-# initialize the client
-client = SerpApi::Client.new(adapter: :async_http, api_key: ENV['API_KEY'], timeout: 10)
-
-# run same query in parallel
-Async do
-  Async do
-    data = client.search(engine: 'google', q: 'Coffee', location: 'Austin, TX')
-    expect(data.keys.size).to be > 5
-    expect(data.dig(:search_metadata,:id)).not_to be_nil
-  end
-  Async do
-    data = client.search(engine: 'youtube', search_query: 'Coffee', location: 'Austin, TX')
-    expect(data.keys.size).to be > 5
-    expect(data.dig(:search_metadata,:id)).not_to be_nil
-  end
-end
-```
-
 ## Basic example per search engine
 
 Here is how to calls the APIs.
@@ -841,15 +800,47 @@ Search API enables `async` search.
 Here is an example of asynchronous searches using Ruby 
 ```ruby
 require 'serpapi'
+# The code snippet aims to improve the efficiency of searching using the SerpApi client async function. It 
+# targets companies in the MAANG (Meta, Amazon, Apple, Netflix, Google) group.
+#
+# **Process:**
+# 1. **Request Queue:** The company list is iterated over, and each company is queried using the SerpApi client. Requests 
+# are stored in a queue to avoid blocking the main thread.
+#
+# 2. **Client Retrieval:** After each request, the code checks the status of the search result. If it's cached or 
+# successful, the company name is printed, and the request is skipped. Otherwise, the result is added to the queue for 
+# further processing.
+#
+# 3. **Queue Processing:** The queue is processed until it's empty. In each iteration, the last result is retrieved and 
+# its client ID is extracted.
+#
+# 4. **Archived Client Retrieval:** Using the client ID, the code retrieves the archived client and checks its status. If 
+# it's cached or successful, the company name is printed, and the client is skipped. Otherwise, the result is added back 
+# to the queue for further processing.
+#
+# 5. **Completion:** The queue is closed, and a message is printed indicating that the process is complete.
+#
+# * **Asynchronous Requests:** The `async: true` option ensures that search requests are processed in parallel, improving 
+# efficiency.
+# * **Queue Management:** The queue allows requests to be processed asynchronously without blocking the main thread.
+# * **Status Checking:** The code checks the status of each search result before processing it, avoiding unnecessary work.
+# * **Queue Processing:** The queue ensures that all requests are processed in the order they were submitted.
+
+# **Overall, the code snippet demonstrates a well-structured approach to improve the efficiency of searching for company 
+# information using SerpApi.**
+
+# load serpapi library
+require 'serpapi'
+
 # target MAANG companies
 company_list = %w(meta amazon apple netflix google)
-client = SerpApi::Client.new({engine: 'google', async: true, api_key: ENV['API_KEY']})
+client = SerpApi::Client.new(engine: 'google', async: true, persistent: true, api_key: ENV['API_KEY'])
 search_queue = Queue.new
 company_list.each do |company|
   # store request into a search_queue - no-blocker
   result = client.search({q: company})
   if result[:search_metadata][:status] =~ /Cached|Success/
-    puts "#{company}: client done"
+    puts "#{company}: search results found in cache for: #{company}"
     next
   end
 
@@ -864,16 +855,17 @@ while !search_queue.empty?
   search_id = result[:search_metadata][:id]
 
   # retrieve client from the archive - blocker
-  search_archived = client.2(search_id)
+  search_archived = client.search_archive(search_id)
   if search_archived[:search_metadata][:status] =~ /Cached|Success/
-    puts "#{search_archived[:search_parameters][:q]}: client done"
+    puts "#{search_archived[:search_parameters][:q]}: search results found in archive for: #{company}"
     next
   end
 
   # add results to the client queue
   search_queue.push(result)
 end
 
+# destroy the queue
 search_queue.close
 puts 'done'```
 

README.md.erb

@@ -179,47 +179,6 @@ pp client.account
 
 It prints your account information.
 
-### Change HTTP client implementation
-
-The internal HTTP client is based on the [Faraday](https://lostisland.github.io/faraday/) library.
- You can change the HTTP client by passing the adapter as an argument to the `SerpApi::Client.new` method.
-
-It requires to install the adapter gem.
-```sh
-gem install faraday-httpclient
-```
-
-```ruby
-# see faraday documentation for adapter description.
-client = SerpApi::Client.new(engine: 'google', api_key: ENV['API_KEY'], timeout: 10, adapter: :httpclient)
-data = client.search(q: 'Coffee', location: 'Austin, TX')
-pp data
-```
-
-or a more [advanced parallel requests example](https://lostisland.github.io/faraday/#/advanced/parallel-requests)
-
-```ruby
-# install and load
-require 'async/http/faraday'
-
-# initialize the client
-client = SerpApi::Client.new(adapter: :async_http, api_key: ENV['API_KEY'], timeout: 10)
-
-# run same query in parallel
-Async do
-  Async do
-    data = client.search(engine: 'google', q: 'Coffee', location: 'Austin, TX')
-    expect(data.keys.size).to be > 5
-    expect(data.dig(:search_metadata,:id)).not_to be_nil
-  end
-  Async do
-    data = client.search(engine: 'youtube', search_query: 'Coffee', location: 'Austin, TX')
-    expect(data.keys.size).to be > 5
-    expect(data.dig(:search_metadata,:id)).not_to be_nil
-  end
-end
-```
-
 ## Basic example per search engine
 
 Here is how to calls the APIs.

Rakefile

@@ -63,6 +63,8 @@ end
 desc 'run demo example'
 task :demo do
   sh 'ruby oobt/demo.rb'
+  sh 'ruby oobt/demo_async.rb'
+  sh 'ruby oobt/demo_thread_pool.rb'
 end
 
 desc 'release the gem to the public rubygems.org'

lib/serpapi.rb

@@ -4,8 +4,8 @@ module SerpApi
   # see serpapi for implementation
 end
 
-# load faraday HTTP lib
-require 'faraday'
+# load HTTP
+require 'http'
 
 # load
 require 'json'

lib/serpapi/client.rb

@@ -4,15 +4,18 @@ module SerpApi
   # Client for SerpApi.com
   #
   class Client
-    include Errors
 
     # Backend service URL
     BACKEND = 'serpapi.com'.freeze
 
     # HTTP timeout requests
     attr_reader :timeout,
                 # Query parameters
-                :params
+                :params,
+                # HTTP persistent
+                :persistent,
+                # HTTP.rb client
+                :socket
 
     # Constructor
     #
@@ -30,21 +33,32 @@ class Client
     # key can be either a symbol or a string.
     #
     # @param [Hash] params default for the search
-    def initialize(params = {}, _adapter = :net_http)
-      # set default read timeout
-      @timeout = params[:timeout] || params['timeout'] || 120
+    def initialize(params = {})
+      # store client HTTP request timeout
+      @timeout = params[:timeout] || 120
       @timeout.freeze
 
+      # enable HTTP persistent mode
+      @persistent = params[:persistent] || true
+      @persistent.freeze
+
       # delete this client only configuration keys
-      params.delete('timeout') if params.key? 'timeout'
-      params.delete(:timeout) if params.key? :timeout
+      %i(timeout persistent).each do |option|
+        params.delete(option) if params.key?(option)
+      end
 
-      # set default params safely in memory
+      # set default serpapi related parameters
       @params = params.clone || {}
+
+      # track ruby library as a client for statistic purpose
+      @params[:source] = 'serpapi-ruby:' << SerpApi::VERSION
+
       @params.freeze
 
-      # setup connection socket
-      @socket = Faraday.new(url: "https://#{BACKEND}")
+      # create connection socket
+      if persistent?
+        @socket = HTTP.persistent("https://#{BACKEND}")
+      end
     end
 
     # perform a search using SerpApi.com
@@ -83,7 +97,7 @@ def location(params = {})
     # @param [Symbol] format :json or :html (default: json, optional)
     # @return [String|Hash] raw html or JSON / Hash
     def search_archive(search_id, format = :json)
-      raise SerpApiException, 'format must be json or html' unless [:json, :html].include?(format)
+      raise SerpApiError, 'format must be json or html' unless [:json, :html].include?(format)
 
       get("/searches/#{search_id}.#{format}", format)
     end
@@ -105,46 +119,60 @@ def api_key
       @params[:api_key]
     end
 
+    # close open connection 
+    def close
+      @socket.close if @socket
+    end
+
     private
 
     # @return [Hash] query parameter
     def query(params)
       # merge default params with custom params
       q = @params.merge(params)
 
-      # set ruby client
-      q[:source] = 'serpapi-ruby:' << SerpApi::VERSION
-
       # delete empty key/value
       q.compact
     end
 
+    # @return [Boolean] HTTP session persistent enabled
+    def persistent?
+      persistent
+    end
+
     # get HTTP query formatted results
     #
     # @param [String] endpoint HTTP service uri
     # @param [Symbol] decoder type :json or :html
     # @param [Hash] params custom search inputs
     # @param [Boolean] symbolize_names if true, convert JSON keys to symbols
-    # @return decoded payload as JSON / Hash or String
+    # @return decoded response as JSON / Hash or String
     def get(endpoint, decoder = :json, params = {}, symbolize_names = true)
-      payload = @socket.get(endpoint) do |req|
-        req.params = query(params)
-        req.options.timeout = timeout
+      # execute get via open socket 
+      if persistent?
+        response = @socket.get(endpoint, params: query(params))
+      else
+        response = HTTP.timeout(timeout).get("https://#{BACKEND}#{endpoint}", params: query(params))
       end
-      # read http response
-      data = payload.body
-      # decode payload using JSON native parser
-      if decoder == :json
-        data = JSON.parse(data, symbolize_names: symbolize_names)
-        if data.instance_of?(Hash) && data.key?('error')
-          raise SerpApiException, "get failed with error: #{data['error']} from url: #{endpoint}, params: #{params}, decoder: #{decoder}, http status: #{payload.status} "
+      
+      # decode response using JSON native parser
+      case decoder
+      when :json
+        # read http response
+        data = JSON.parse(response.body, symbolize_names: symbolize_names)
+        if data.instance_of?(Hash) && data.key?(:error)
+          raise SerpApiError, "HTTP request failed with error: #{data[:error]} from url: https://#{BACKEND}#{endpoint}, params: #{params}, decoder: #{decoder}, response status: #{response.status} "
+        elsif response.status != 200
+          raise SerpApiError, "HTTP request failed with response status: #{response.status} reponse: #{data} on get url: https://#{BACKEND}#{endpoint}, params: #{params}, decoder: #{decoder}" 
         end
-        raise SerpApiException, "get failed with response status: #{payload.status} reponse: #{data} on get url: #{endpoint}, params: #{params}, decoder: #{decoder}" if payload.status != 200
+
+        # discard response body
+        response.flush if persistent?
+
+        return data
+      else
+        return response.body
       end
-      # return raw HTML
-      data
-    rescue Faraday::Error => e
-      raise SerpApiException, "fail: get url: #{endpoint} caused by #{e.class} : #{e.message} (params: #{params}, decoder: #{decoder})"
     end
   end
 end

lib/serpapi/error.rb

@@ -1,12 +1,11 @@
 # Module includes SerpApi exception
-#
-module SerpApi
-  # Errors module contains all the exceptions used in the SerpApi client.
-  #
-  module Errors
-    #  SerpApiException wraps anything related to the SerpApi client errors.
-    #
-    class SerpApiException < StandardError
-    end
+module SerpApi  
+  # SerpApiException wraps any errors related to the SerpApi client.
+  class SerpApiError < StandardError
+    # List the specific types of errors handled by the Error class.
+    #  - Missing API key
+    #  - Credit limit
+    #  - Incorrect query
+    #  - more ...
   end
 end