Merge pull request #10 from proxymesh/feature/env-var-config

Update examples to use environment variables for proxy config

Author: proxymesh

README.md

@@ -17,6 +17,34 @@ The [python-proxy-headers](https://github.com/proxymesh/python-proxy-headers) li
 pip install python-proxy-headers
 ```
 
+**Running Examples:**
+
+All examples read proxy configuration from environment variables:
+
+```bash
+# Required: Set your proxy URL
+export PROXY_URL='http://user:pass@proxy.example.com:8080'
+
+# Optional: Custom test URL (default: https://api.ipify.org?format=json)
+export TEST_URL='https://httpbin.org/ip'
+
+# Optional: Send a custom header to the proxy
+export PROXY_HEADER='X-ProxyMesh-Country'
+export PROXY_VALUE='US'
+
+# Optional: Read a specific header from the response
+export RESPONSE_HEADER='X-ProxyMesh-IP'
+
+# Run a single example
+python python/requests-proxy-headers.py
+
+# Run all examples as tests
+python python/run_tests.py
+
+# Run specific examples
+python python/run_tests.py requests-proxy-headers httpx-proxy-headers
+```
+
 **Examples:**
 
 | Library | Example | Description |

python/aiohttp-proxy-headers.py

@@ -2,32 +2,46 @@
 """
 aiohttp with proxy headers example.
 
-This example shows how to send custom headers to a proxy server and
-receive proxy response headers using the python-proxy-headers library
-with async/await.
+Configuration via environment variables:
+    PROXY_URL       - Proxy URL (required), e.g., http://user:pass@proxy:8080
+    TEST_URL        - URL to request (default: https://api.ipify.org?format=json)
+    PROXY_HEADER    - Header name to send to proxy (optional)
+    PROXY_VALUE     - Header value to send to proxy (optional)
+    RESPONSE_HEADER - Header name to read from response (optional)
 
 See: https://github.com/proxymesh/python-proxy-headers
-Docs: https://python-proxy-headers.readthedocs.io/en/latest/aiohttp.html
 """
+import os
+import sys
 import asyncio
 from python_proxy_headers import aiohttp_proxy
 
+# Get configuration from environment
+proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY')
+if not proxy_url:
+    print("Error: Set PROXY_URL environment variable", file=sys.stderr)
+    sys.exit(1)
+
+test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json')
+proxy_header = os.environ.get('PROXY_HEADER')
+proxy_value = os.environ.get('PROXY_VALUE')
+response_header = os.environ.get('RESPONSE_HEADER')
+
+proxy_headers = {proxy_header: proxy_value} if proxy_header and proxy_value else None
+
 
 async def main():
     async with aiohttp_proxy.ProxyClientSession() as session:
         async with session.get(
-            'https://api.ipify.org?format=json',
-            proxy='http://USERNAME:PASSWORD@PROXYHOST:PORT',
-            proxy_headers={'X-ProxyMesh-Country': 'US'}
+            test_url,
+            proxy=proxy_url,
+            proxy_headers=proxy_headers
         ) as response:
-            # Print response
-            print(f"Status: {response.status}")
             body = await response.text()
+            print(f"Status: {response.status}")
             print(f"Body: {body}")
-            
-            # Access proxy response headers (from CONNECT response)
-            proxy_ip = response.headers.get('X-ProxyMesh-IP')
-            print(f"Proxy IP: {proxy_ip}")
+            if response_header:
+                print(f"{response_header}: {response.headers.get(response_header)}")
 
 
 if __name__ == '__main__':

python/autoscraper-proxy-headers.py

@@ -2,43 +2,44 @@
 """
 AutoScraper with proxy headers example.
 
-This example shows how to use AutoScraper with custom proxy headers.
-AutoScraper automatically learns scraping rules while python-proxy-headers
-enables sending custom headers to the proxy server.
+Configuration via environment variables:
+    PROXY_URL       - Proxy URL (required), e.g., http://user:pass@proxy:8080
+    TEST_URL        - URL to request (default: https://api.ipify.org?format=json)
+    PROXY_HEADER    - Header name to send to proxy (optional)
+    PROXY_VALUE     - Header value to send to proxy (optional)
+    RESPONSE_HEADER - Header name to read from response (optional)
 
 See: https://github.com/proxymesh/python-proxy-headers
-Docs: https://python-proxy-headers.readthedocs.io/en/latest/autoscraper.html
 """
+import os
+import sys
 from python_proxy_headers.autoscraper_proxy import ProxyAutoScraper
 
-# Create an AutoScraper with proxy header support
-scraper = ProxyAutoScraper(proxy_headers={'X-ProxyMesh-Country': 'US'})
-
-# Proxy configuration
-proxies = {
-    'http': 'http://USERNAME:PASSWORD@PROXYHOST:PORT',
-    'https': 'http://USERNAME:PASSWORD@PROXYHOST:PORT'
-}
-
-# Build scraping rules from a sample page
-# AutoScraper learns what to extract based on the wanted_list
-result = scraper.build(
-    url='https://quotes.toscrape.com/',
-    wanted_list=['The world as we have created it is a process of our thinking.'],
-    request_args={'proxies': proxies}
-)
-
-print(f"Found {len(result)} matching quotes")
-for quote in result[:5]:
-    print(f"  - {quote[:60]}...")
-
-# Save learned rules for later use
-# scraper.save('quotes_rules.json')
-
-# Use learned rules on another page
-# result = scraper.get_result_similar(
-#     url='https://quotes.toscrape.com/page/2/',
-#     request_args={'proxies': proxies}
-# )
+# Get configuration from environment
+proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY')
+if not proxy_url:
+    print("Error: Set PROXY_URL environment variable", file=sys.stderr)
+    sys.exit(1)
+
+test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json')
+proxy_header = os.environ.get('PROXY_HEADER')
+proxy_value = os.environ.get('PROXY_VALUE')
+response_header = os.environ.get('RESPONSE_HEADER')
+
+proxy_headers = {proxy_header: proxy_value} if proxy_header and proxy_value else None
+
+# Create scraper and test via underlying session
+scraper = ProxyAutoScraper(proxy_headers=proxy_headers)
+session = scraper._get_session()
+session.proxies = {'http': proxy_url, 'https': proxy_url}
+
+# Make request
+response = session.get(test_url)
+
+# Output
+print(f"Status: {response.status_code}")
+print(f"Body: {response.text}")
+if response_header:
+    print(f"{response_header}: {response.headers.get(response_header)}")
 
 scraper.close()

python/cloudscraper-proxy-headers.py

@@ -2,34 +2,41 @@
 """
 CloudScraper with proxy headers example.
 
-This example shows how to use CloudScraper with custom proxy headers.
-CloudScraper bypasses Cloudflare protection while python-proxy-headers
-enables sending/receiving custom proxy headers.
+Configuration via environment variables:
+    PROXY_URL       - Proxy URL (required), e.g., http://user:pass@proxy:8080
+    TEST_URL        - URL to request (default: https://api.ipify.org?format=json)
+    PROXY_HEADER    - Header name to send to proxy (optional)
+    PROXY_VALUE     - Header value to send to proxy (optional)
+    RESPONSE_HEADER - Header name to read from response (optional)
 
 See: https://github.com/proxymesh/python-proxy-headers
-Docs: https://python-proxy-headers.readthedocs.io/en/latest/cloudscraper.html
 """
+import os
+import sys
 from python_proxy_headers.cloudscraper_proxy import create_scraper
 
-# Create a CloudScraper with proxy header support
-scraper = create_scraper(
-    proxy_headers={'X-ProxyMesh-Country': 'US'},
-    browser='chrome'
-)
+# Get configuration from environment
+proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY')
+if not proxy_url:
+    print("Error: Set PROXY_URL environment variable", file=sys.stderr)
+    sys.exit(1)
 
-# Set proxy
-scraper.proxies = {
-    'http': 'http://USERNAME:PASSWORD@PROXYHOST:PORT',
-    'https': 'http://USERNAME:PASSWORD@PROXYHOST:PORT'
-}
+test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json')
+proxy_header = os.environ.get('PROXY_HEADER')
+proxy_value = os.environ.get('PROXY_VALUE')
+response_header = os.environ.get('RESPONSE_HEADER')
 
-# Make request - Cloudflare bypass + proxy headers work together
-response = scraper.get('https://api.ipify.org?format=json')
+proxy_headers = {proxy_header: proxy_value} if proxy_header and proxy_value else None
 
-# Print response
+# Create scraper with proxy headers
+scraper = create_scraper(proxy_headers=proxy_headers, browser='chrome')
+scraper.proxies = {'http': proxy_url, 'https': proxy_url}
+
+# Make request
+response = scraper.get(test_url)
+
+# Output
 print(f"Status: {response.status_code}")
 print(f"Body: {response.text}")
-
-# Access proxy response headers
-proxy_ip = response.headers.get('X-ProxyMesh-IP')
-print(f"Proxy IP: {proxy_ip}")
+if response_header:
+    print(f"{response_header}: {response.headers.get(response_header)}")

python/httpx-async-proxy-headers.py

@@ -2,37 +2,51 @@
 """
 httpx async with proxy headers example.
 
-This example shows how to send custom headers to a proxy server and
-receive proxy response headers using httpx's async client.
+Configuration via environment variables:
+    PROXY_URL       - Proxy URL (required), e.g., http://user:pass@proxy:8080
+    TEST_URL        - URL to request (default: https://api.ipify.org?format=json)
+    PROXY_HEADER    - Header name to send to proxy (optional)
+    PROXY_VALUE     - Header value to send to proxy (optional)
+    RESPONSE_HEADER - Header name to read from response (optional)
 
 See: https://github.com/proxymesh/python-proxy-headers
-Docs: https://python-proxy-headers.readthedocs.io/en/latest/httpx.html
 """
+import os
+import sys
 import asyncio
 import httpx
 from python_proxy_headers.httpx_proxy import AsyncHTTPProxyTransport
 
+# Get configuration from environment
+proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY')
+if not proxy_url:
+    print("Error: Set PROXY_URL environment variable", file=sys.stderr)
+    sys.exit(1)
+
+test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json')
+proxy_header = os.environ.get('PROXY_HEADER')
+proxy_value = os.environ.get('PROXY_VALUE')
+response_header = os.environ.get('RESPONSE_HEADER')
+
+proxy_headers = {proxy_header: proxy_value} if proxy_header and proxy_value else None
+
 
 async def main():
-    # Create a proxy with custom headers
-    proxy = httpx.Proxy(
-        'http://USERNAME:PASSWORD@PROXYHOST:PORT',
-        headers={'X-ProxyMesh-Country': 'US'}
-    )
+    # Create proxy with optional headers
+    if proxy_headers:
+        proxy = httpx.Proxy(proxy_url, headers=proxy_headers)
+    else:
+        proxy = proxy_url
 
-    # Create async transport with proxy header support
     transport = AsyncHTTPProxyTransport(proxy=proxy)
 
     async with httpx.AsyncClient(mounts={'http://': transport, 'https://': transport}) as client:
-        response = await client.get('https://api.ipify.org?format=json')
+        response = await client.get(test_url)
 
-        # Print response
         print(f"Status: {response.status_code}")
         print(f"Body: {response.text}")
-        
-        # Access proxy response headers (from CONNECT response)
-        proxy_ip = response.headers.get('X-ProxyMesh-IP')
-        print(f"Proxy IP: {proxy_ip}")
+        if response_header:
+            print(f"{response_header}: {response.headers.get(response_header)}")
 
 
 if __name__ == '__main__':

python/httpx-proxy-headers.py

@@ -2,28 +2,44 @@
 """
 httpx with proxy headers example.
 
-This example shows how to send custom headers to a proxy server and
-receive proxy response headers using the python-proxy-headers library.
+Configuration via environment variables:
+    PROXY_URL       - Proxy URL (required), e.g., http://user:pass@proxy:8080
+    TEST_URL        - URL to request (default: https://api.ipify.org?format=json)
+    PROXY_HEADER    - Header name to send to proxy (optional)
+    PROXY_VALUE     - Header value to send to proxy (optional)
+    RESPONSE_HEADER - Header name to read from response (optional)
 
 See: https://github.com/proxymesh/python-proxy-headers
-Docs: https://python-proxy-headers.readthedocs.io/en/latest/httpx.html
 """
+import os
+import sys
 import httpx
 from python_proxy_headers import httpx_proxy
 
-# Create a proxy with custom headers
-proxy = httpx.Proxy(
-    'http://USERNAME:PASSWORD@PROXYHOST:PORT',
-    headers={'X-ProxyMesh-Country': 'US'}
-)
+# Get configuration from environment
+proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY')
+if not proxy_url:
+    print("Error: Set PROXY_URL environment variable", file=sys.stderr)
+    sys.exit(1)
 
-# Make a request using the helper function
-response = httpx_proxy.get('https://api.ipify.org?format=json', proxy=proxy)
+test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json')
+proxy_header = os.environ.get('PROXY_HEADER')
+proxy_value = os.environ.get('PROXY_VALUE')
+response_header = os.environ.get('RESPONSE_HEADER')
 
-# Print response
+proxy_headers = {proxy_header: proxy_value} if proxy_header and proxy_value else None
+
+# Create proxy with optional headers
+if proxy_headers:
+    proxy = httpx.Proxy(proxy_url, headers=proxy_headers)
+else:
+    proxy = proxy_url
+
+# Make request
+response = httpx_proxy.get(test_url, proxy=proxy)
+
+# Output
 print(f"Status: {response.status_code}")
 print(f"Body: {response.text}")
-
-# Access proxy response headers (from CONNECT response)
-proxy_ip = response.headers.get('X-ProxyMesh-IP')
-print(f"Proxy IP: {proxy_ip}")
+if response_header:
+    print(f"{response_header}: {response.headers.get(response_header)}")