??: chatGPT/LLM? ??? ????

??? ????? ?? ? ???? ???? ???? ?????? ??? ???? ???? ????? ???? ?????. ???? ?? ??? ????? ??? ?? ??? ?? ???? ???? ???? ?? ??? ????. ??? ???? ?? ????? ???? ???? ??? ? ??? ? ?? ????.

??? ??? ?? ?? ??? ??? ? Scrapy? ?? ???? ???? ??????? ??? ??? ??? ??? ? ?? ??? ???? ????. ??? ??? ???? ???? ??? requests ? beautifulsoup? ?? ?? ??? ???? ?????? ??? ????. ??? ?? ??? ???? ???? ????? ??? ??? ? ??? ?????? Pydantic? ??? ??? ??? ?? ??? ????? ?? ??? ?? ??? ?????.
https://docs.pydantic.dev/latest/
Pydantic? ??? ?? Python ?????. ?? ?? ?? API ??? FastAPI? ????? ???. Pydantic? ????? ??? ???? ? ??? ??? ??? ? ?? ?? Python ??? ????. ??? ? ?????? pydantic? ???? ??? ?? ??? ??? ????(?? ???? ?? ??? pydantic? ??? ? ? ????)

• Cerberus? ??? ?? ??? Python? ??? ?? ????????. https://pypi.org/project/Cerberus/

???? ??:

?? ?????? ?? ???? ??? ???? ????.
??? beautifulsoup? ???? ???? ?????. pydantic ??? ???? ???? ???? ? ???? ???? ?????. ????? ???? ??? ???? json ??? ?????.

? ?? ??? ??? ?? ? ??? ?? ???? ??? ? ?? Python ???? ???????.

?? ????

import requests # for web request
from bs4 import BeautifulSoup # cleaning html content

# pydantic for validation

from pydantic import BaseModel, field_validator, ValidationError

import json

1. ?? ??? ? ?? ??

??? ???? http://quotes.toscrape.com/)? ???? ????. ? ????? quote_text, ??? ? ??? ? ?? ??? ????. ?:

?? ??? ?? URL? ?? HTML ???? ???? ???? ???????.

def get_html_content(page_url: str) -> str:
    page_content =""
    # Send a GET request to the website
    response = requests.get(url)
    # Check if the request was successful (status code 200)
    if response.status_code == 200:
        page_content = response.content
    else:
        page_content = f'Failed to retrieve the webpage. Status code: {response.status_code}'
    return page_content

2. ?????? ?? ??? ????

??? beautifulsoup? ???? ??? URL?? ???? ??????. ????? ? ???? ????. 1) ??? HTML ??? ???? 2) ? ?? ??? ?? ??? HTML ?? ?? 3) ? ???? ? ????

import requests # for web request
from bs4 import BeautifulSoup # cleaning html content

# pydantic for validation

from pydantic import BaseModel, field_validator, ValidationError

import json

def get_html_content(page_url: str) -> str:
    page_content =""
    # Send a GET request to the website
    response = requests.get(url)
    # Check if the request was successful (status code 200)
    if response.status_code == 200:
        page_content = response.content
    else:
        page_content = f'Failed to retrieve the webpage. Status code: {response.status_code}'
    return page_content

?? ????? ? ???? div?? ??? ???? ?????.

def get_tags(tags):
    tags =[tag.get_text() for tag in tags.find_all('a')]
    return tags

3. Pydantic ??? ???? ???? ? ??? ?? ??? ???? ?????.

??? ? ???? pydantic ???? ???? ??? ???? ? ??? ??? ?? ??? ???? ?????.

pydantic ?? ???

??? quote_text, ??? ? ??? ?? ? ?? ??? ?? BaseModel?? ??? Quote ??????. ? 3?? ? quote_text? ???? ???(str) ????, ??? ?? ?????.

? ?? ??? ??? ???(????? ??)? ????.

1) tagged_more_than_two () : ??? 2? ???? ?????. (?? ?? ???? ?? ????? ??? ? ????)

2.) check_quote_text(): ? ???? ????? ""? ???? ???? ??????.

def get_quotes_div(html_content:str) -> str :    
    # Parse the page content with BeautifulSoup
    soup = BeautifulSoup(html_content, 'html.parser')

    # Find all the quotes on the page
    quotes = soup.find_all('div', class_='quote')

    return quotes

??? ???? ? ??? ??

pydantic? ???? ??? ??? ??? ?? ????. ?? ?? ?? ????? ???? ???? pydantic ??? Quote? ?????.

    # Loop through each quote and extract the text and author
    for quote in quotes_div:
        quote_text = quote.find('span', class_='text').get_text()
        author = quote.find('small', class_='author').get_text()
        tags = get_tags(quote.find('div', class_='tags'))

        # yied data to a dictonary 
        quote_temp ={'quote_text': quote_text,
                'author': author,
                'tags':tags
        }

class Quote(BaseModel):
    quote_text:str
    author:str
    tags: list

    @field_validator('tags')
    @classmethod
    def tags_more_than_two(cls, tags_list:list) -> list:
        if len(tags_list) <=2:
            raise ValueError("There should be more than two tags.")
        return tags_list

    @field_validator('quote_text')
    @classmethod    
    def check_quote_text(cls, quote_text:str) -> str:
        return quote_text.removeprefix('“').removesuffix('”')

4. ??? ??

???? ???? json ??? ?????. (Python ??? json ??? ???? ?? ???? ???????.)

quote_data = Quote(**quote_temp)

?? ???

????? ? ??? ????? ?? ?? ??? ????? ???? ???? ??? ? ????.

def get_quotes_data(quotes_div: list) -> list:
    quotes_data = []

    # Loop through each quote and extract the text and author
    for quote in quotes_div:
        quote_text = quote.find('span', class_='text').get_text()
        author = quote.find('small', class_='author').get_text()
        tags = get_tags(quote.find('div', class_='tags'))

        # yied data to a dictonary 
        quote_temp ={'quote_text': quote_text,
                'author': author,
                'tags':tags
        }

        # validate data with Pydantic model
        try:
            quote_data = Quote(**quote_temp)            
            quotes_data.append(quote_data.model_dump())            
        except  ValidationError as e:
            print(e.json())
    return quotes_data

??: ??? ???? ????. ??? ??? ??? ????? ??? ?????.

?? ? ???:

• https://pypi.org/project/parsel/

• https://docs.pydantic.dev/latest/

? ??? ??? ? ??: Pydantic ??? ??? ??? ???? ?? ?????. ??? ??? PHP ??? ????? ?? ?? ??? ?????!