Beginner's Guide to Web Scraping with BeautifulSoup in Python

1pip install beautifulsoup4
2

1from bs4 import BeautifulSoup
2import requests
3

1url = 'https://example.com'
2response = requests.get(url)
3

1soup = BeautifulSoup(response.text, 'html.parser')
2

1# Find all <a> tags
2links = soup.find_all('a')
3
4# Extract text from the first <a> tag
5first_link_text = links[0].text
6
7# Extract the value of a specific attribute (e.g., href)
8first_link_href = links[0]['href']
9

1# Find an element by class
2element_by_class = soup.find('div', class_='example-class')
3
4# Find an element by ID
5element_by_id = soup.find('div', id='example-id')
6

1# Extract text from a specific element
2element_text = element_by_class.text
3
4# Extract text from multiple elements
5elements_texts = [element.text for element in soup.find_all('p')]
6

1# Extract text from all <p> tags within a <div> with class 'container'
2p_tags_in_div = soup.select('.container p')
3

1from bs4 import BeautifulSoup
2import requests
3
4# Fetch the webpage
5url = 'https://example.com'
6response = requests.get(url)
7
8# Parse the webpage
9soup = BeautifulSoup(response.text, 'html.parser')
10
11# Extract all links
12links = soup.find_all('a')
13
14# Print the text of each link
15for link in links:
16    print(link.text)
17