olivia_finder.myrequests.proxy_builders.list_builder
1from typing import List 2import requests 3from typing_extensions import override 4from .proxy_builder import ProxyBuilder 5 6 7class ListProxyBuilder(ProxyBuilder): 8 ''' 9 A generic proxy builder implementation, gets the proxies from a list in the format ip:port 10 11 Parameters 12 ---------- 13 url : str 14 URL of the proxy list website to get the proxies 15 request_timeout : int 16 Timeout for the proxy list requests 17 18 Raises 19 ------ 20 ValueError 21 If url parameter is None 22 ''' 23 24 TIMEOUT = 60 25 26 def __init__(self, url:str, request_timeout: int = TIMEOUT): 27 ''' 28 Constructor 29 ''' 30 31 super().__init__( 32 url=url, 33 request_timeout=request_timeout 34 ) 35 36 @override 37 def _parse_request(self, response: requests.Response) -> List[str]: 38 ''' 39 Parse the response and return a list of proxies 40 Expected data: 41 42 81.4.102.223:8081 43 35.247.228.52:3129 44 20.121.242.93:3128 45 35.247.234.213:3129 46 35.247.248.45:3129 47 35.247.248.46:3129 48 35.247.240.117:3129 49 35.247.254.240:3129 50 35.247.197.252:3129 51 35.247.246.68:3129 52 53 Parameters 54 ---------- 55 response : requests.Response 56 The response of the request 57 58 Returns 59 ------- 60 List[str] 61 A list of proxies 62 ''' 63 64 data = response.text.splitlines() 65 return [] if data is None else data 66
8class ListProxyBuilder(ProxyBuilder): 9 ''' 10 A generic proxy builder implementation, gets the proxies from a list in the format ip:port 11 12 Parameters 13 ---------- 14 url : str 15 URL of the proxy list website to get the proxies 16 request_timeout : int 17 Timeout for the proxy list requests 18 19 Raises 20 ------ 21 ValueError 22 If url parameter is None 23 ''' 24 25 TIMEOUT = 60 26 27 def __init__(self, url:str, request_timeout: int = TIMEOUT): 28 ''' 29 Constructor 30 ''' 31 32 super().__init__( 33 url=url, 34 request_timeout=request_timeout 35 ) 36 37 @override 38 def _parse_request(self, response: requests.Response) -> List[str]: 39 ''' 40 Parse the response and return a list of proxies 41 Expected data: 42 43 81.4.102.223:8081 44 35.247.228.52:3129 45 20.121.242.93:3128 46 35.247.234.213:3129 47 35.247.248.45:3129 48 35.247.248.46:3129 49 35.247.240.117:3129 50 35.247.254.240:3129 51 35.247.197.252:3129 52 35.247.246.68:3129 53 54 Parameters 55 ---------- 56 response : requests.Response 57 The response of the request 58 59 Returns 60 ------- 61 List[str] 62 A list of proxies 63 ''' 64 65 data = response.text.splitlines() 66 return [] if data is None else data
A generic proxy builder implementation, gets the proxies from a list in the format ip:port
Parameters
- url (str): URL of the proxy list website to get the proxies
- request_timeout (int): Timeout for the proxy list requests
Raises
- ValueError: If url parameter is None