olivia_finder.data_source.repository_scrapers.pypi

  1import re
  2from typing import List, Optional
  3import requests
  4from typing_extensions import override
  5from bs4 import BeautifulSoup
  6from ..scraper_ds import ScraperDataSource
  7from ...myrequests.request_handler import RequestHandler
  8from ...myrequests.job import RequestJob
  9from ...utilities.exception import OliviaFinderException
 10
 11class PypiScraper(ScraperDataSource):
 12    ''' 
 13    Class that scrapes the PyPI website to obtain information about Python packages
 14    Implements the abstract class Scraper and accordingly DataSource class
 15    
 16    Attributes
 17    ----------
 18    PYPI_PACKAGE_LIST_URL : str
 19        URL of the PyPI website where the list of packages is located
 20    PYPI_PACKAGE_DATA_URL : str
 21        URL of the PyPI website where the data of a package is located
 22    '''
 23
 24    # Class variables  
 25    PYPI_PACKAGE_LIST_URL: str  = "https://pypi.org/simple/"
 26    PYPI_PACKAGE_DATA_URL: str  = "https://pypi.org/pypi/"
 27
 28
 29    def __init__(self, request_handler: Optional[RequestHandler] = None):
 30        '''
 31        Constructor
 32        '''
 33
 34        super().__init__(request_handler)
 35
 36    @override
 37    def obtain_package_names(self) -> List[str]:
 38        '''
 39        Obtain the list of packages names from the PyPI website
 40        Implements the abstract method of DataSource class
 41
 42        Returns
 43        -------
 44        List[str]
 45            List of packages names
 46            
 47        Handles
 48        -------
 49        Exception
 50            If there is an error obtaining the list of packages, it returns an empty list
 51            
 52        Example
 53        -------
 54        >>> pypi_scraper = PypiScraper()
 55        >>> pypi_scraper.obtain_package_names()
 56        ['package1', 'package2', ...]
 57        '''
 58
 59        # Build the request job
 60        job = RequestJob("PYPI package names", self.PYPI_PACKAGE_LIST_URL)
 61
 62        # Get the HTML of the page
 63        job = self.request_handler.do_request(job)
 64
 65        if job.response is None:
 66            raise OliviaFinderException(f'Error obtaining the list of packages from {self.PYPI_PACKAGE_LIST_URL}')
 67        
 68        soup = BeautifulSoup(job.response.text, 'html.parser')
 69        
 70        pakage_list = []
 71        try:
 72            # Get the list of packages
 73            pakage_list = [a.text for a in soup.find_all('a')]
 74        except Exception as e:
 75            self.logger.error(f'Error obtaining the list of packages from {self.PYPI_PACKAGE_LIST_URL}')
 76
 77        self.logger.info(f'Obtained {len(pakage_list)} packages from {self.PYPI_PACKAGE_LIST_URL}')
 78        return pakage_list
 79    
 80    @override
 81    def _build_url(self, package_name: str) -> str:
 82        '''
 83        Build the URL to scrape a package
 84        Implements the abstract method of Scraper class
 85        Parameters
 86        ----------
 87        pkg_name : str
 88            Name of the package
 89
 90        Returns
 91        -------
 92        str
 93            URL to scrape
 94        '''
 95        return f'{self.PYPI_PACKAGE_DATA_URL}{package_name}/json'
 96
 97    @override
 98    def _parser(self, response: requests.Response) -> dict:
 99        '''
100        Parse the JSON data of a package and return the package data as a dictionary
101        
102        Parameters
103        ----------
104        response : requests.Response
105            Response of the request to the package data URL
106        
107        Returns
108        -------
109        dict
110            dictionary with the package data in the following format:
111            {
112                'name': name: str,
113                'version': version: str,
114                'url': url: str,
115                'dependencies': dependencies: List[str]
116            }
117        '''
118        # Parse the JSON
119        data = response.json()
120
121        # Get the dependencies if they exist, build the list of dependencies as Package objects
122
123        dependencies = []
124        if data['info']['requires_dist'] is not None:
125            dependencies_raw = data['info']['requires_dist']
126            
127            # Build a dictionary with the dependencies to avoid duplicates with different versions
128            dependencies_dict = {}
129            
130            for dependency in dependencies_raw:
131                
132                # # Split the dependency in name and version
133                # dependency_data = dependency.split(' ')
134
135                # # check if the dependency has a version
136                # if len(dependency_data) == 1:
137                #     dependencies_Dict[dependency_data[0]] = ""
138                # else:
139                #     # Add the dependency to the dictionary
140                #     # dependency_name: dependency_version
141                #     dependencies_Dict[dependency_data[0]] = dependency_data[1]
142
143                # Get the name of the dependency
144                dependency_name = self._clean_name(dependency)
145                dependencies_dict[dependency_name] = None
146                
147            # Build the list of dependencies as dictionaries
148            dependencies = [{'name': name, 'version': version} for name, version in dependencies_dict.items()]
149            
150            
151        # Build the dictionary and return it
152        return {
153            'name': data['info']['name'],
154            'version': data['info']['version'],
155            'url': data['info']['project_url'],
156            'dependencies': dependencies,
157        }    
158
159    def _clean_name(self, name: str) -> str:
160        '''
161        Clean the package name from versions and other characters
162
163        Parameters
164        ----------
165        name : str
166            Name of the package
167        
168        Returns
169        -------
170        str
171            Cleaned name of the package
172            
173        '''
174        
175        regex = re.compile(r'[^\w\d]+')
176
177        # Reemplazar cualquier coincidencia de la expresión regular en la cadena de dependencia con un espacio en blanco
178        name = regex.sub(' ', name)
179
180        # Obtenga el primer elemento de la lista resultante después de dividir la cadena por espacios en blanco
181        name = name.split()[0]
182
183        return name