Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- from bs4 import BeautifulSoup
- import requests
- url = "https://kalimatimarket.gov.np/#commodityPricesDailyTable"
- # Mimicking as a real person browsing a web from an iphone device, but it's not 100% effective:
- headers = {'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X)'
- ' AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148'}
- req = requests.get(url, headers=headers) # requesting to Kalimati-market server:
- soup = BeautifulSoup(req.content, 'lxml') # making a soup:
- # html tag of the commodity price table, table body and finally tr i.e. table rows:
- table = soup.find('table', id='commodityDailyPrice').find('tbody').find_all('tr')
- # Date of the updated data as it is updated daily and this avoids overwriting of files.
- def date_header():
- # this is the html of table i.e. commodity price table:
- date = soup.find_all('div', id='commodityPricesDailyTable')
- # Looping through table and scrape the day and date in commodity price table:
- for dat in date:
- return dat.find('h5').text
- # function to scrape datas from commodity table: values represents column to scrape:
- # Example: 1 scrape वस्तु datas and so on
- def kalimati_market(value):
- # This technique is list comprehension:
- # It stores pulled data and store in lists: Less code and runs fast: It's quite powerful in Python
- lists = [(tab.find_all('td'))[value].text for tab in table]
- return lists
- # if you don't want to use list-comprehension: Ideally you can use below method:
- # I've commented the function to avoid error. you can delete the above function and use this one if you want to avoid
- # list comprehension
- '''def kalimati_market(value):
- lists = [] # creating empty lists to store values after scraping the needed datas:
- for tab in table: # looping through table and grabbing data from their respective columns
- datas = tab.find_all('td') # this html tag has contains intended datas that we need to pull and save in Excel sheet:
- lists.append(datas[value].text) '''
Add Comment
Please, Sign In to add comment
