Practical practice of climbing Taobao with Selenium
1. Project process
2. Central scheduling
# Central scheduling
Def main ():
Try:
Total = search ()
Total = int (re.compile ('. *? (d +). *?) .search (total) .group (1))
For i in range (2 focus totalizer 1):
Next_page (I)
Except Exception as e:
Print ('exception')
Finally:
Browser.close ()
3. Simulated query
# query based on keywords
Def search ():
Try:
Browser.get ('https://www.taobao.com/')
# until the search box is loaded
Input_search = wait.until (EC.presence_of_element_located ((By.ID,'q')
# until the search button can be clicked
Submit_button = wait.until (EC.element_to_be_clickable ((By.CLASS_NAME,'btn-search')
Input_search.send_keys (KEYWORDS)
Submit_button.click ()
Total = wait.until (EC.presence_of_element_located ((By.CSS_SELECTOR,'div.wraper div.total')
Get_products ()
Return total.text
Except TimeoutException as e:
Print ('response timeout')
4. The operation of the next page
# next page crawl
Def next_page (index):
Try:
Input = wait.until (EC.presence_of_element_located ((By.CSS_SELECTOR,'#mainsrp-pager div.form > input')
Submit = wait.until (EC.element_to_be_clickable ((By.CSS_SELECTOR,'#mainsrp-pager div.form > span.btn.J_Submit')
Input.clear ()
Input.send_keys (index)
Submit.click ()
Wait.until ((By.CSS_SELECTOR,'#mainsrp-pager > div > div > div > ul > li.item.active > span'), str (index)
Get_products ()
Except NoSuchElementException as e:
Print ('element not loaded')
Return next_page (index)
5. Analysis of commodity information.
# get all the goods on one page
Def get_products ():
Wait.until (EC.presence_of_element_located ((By.CSS_SELECTOR, "# mainsrp-itemlist .items"))
Html = browser.page_source
Doc = PyQuery (html)
Items = doc ('.m-itemlist .items. Item') .items ()
For item in items:
If you feel that it is difficult to learn programming, but love the IT industry very much, you can add a test exchange group: 1017539290, join the group to get test learning materials for free!
Product = {
'image':item.find ('.pic-link .img') .attr ('data-src')
'price':float (item.find ('.price'). Text () [2:])
'deal':item.find ('.deal-cnt') .text () [:-3]
'title':item.find ('.title') .text ()
'shop':item.find ('.shop') .text ()
'location':item.find ('.location') .text ()
'keywords':KEYWORDS
}
Save_to_mongo (product=product)
6. Complete code
#! / usr/bin/python
#-*-coding: utf-8-*-
Import pymongo
Import re
From pyquery import PyQuery
From selenium import webdriver
From selenium.webdriver.common.by import By
From selenium.webdriver.support.ui import WebDriverWait
From selenium.webdriver.support import expected_conditions as EC
From selenium.common.exceptions import TimeoutException,NoSuchElementException
From setting import *
Client = pymongo.MongoClient (MONGO_HOST)
Db = client [Mongo _ DB]
Browser = webdriver.Chrome ()
Wait = WebDriverWait (browser,10)
# query based on keywords
Def search ():
Try:
Browser.get ('https://www.taobao.com/')
# until the search box is loaded
Input_search = wait.until (EC.presence_of_element_located ((By.ID,'q')
# until the search button can be clicked
Submit_button = wait.until (EC.element_to_be_clickable ((By.CLASS_NAME,'btn-search')
Input_search.send_keys (KEYWORDS)
Submit_button.click ()
Total = wait.until (EC.presence_of_element_located ((By.CSS_SELECTOR,'div.wraper div.total')
Get_products ()
Return total.text
Except TimeoutException as e:
Print ('response timeout')
# next page crawl if you find it difficult to learn programming, but love the IT industry very much, you can add a test exchange group: 1017539290, join the group to get test learning materials for free!
Def next_page (index):
Try:
Input = wait.until (EC.presence_of_element_located ((By.CSS_SELECTOR,'#mainsrp-pager div.form > input')
Submit = wait.until (EC.element_to_be_clickable ((By.CSS_SELECTOR,'#mainsrp-pager div.form > span.btn.J_Submit')
Input.clear ()
Input.send_keys (index)
Submit.click ()
Wait.until ((By.CSS_SELECTOR,'#mainsrp-pager > div > div > div > ul > li.item.active > span'), str (index)
Get_products ()
Except NoSuchElementException as e:
Print ('element not loaded')
Return next_page (index)
# get all the goods on one page
Def get_products ():
Wait.until (EC.presence_of_element_located ((By.CSS_SELECTOR, "# mainsrp-itemlist .items"))
Html = browser.page_source
Doc = PyQuery (html)
Items = doc ('.m-itemlist .items. Item') .items ()
For item in items:
Product = {
'image':item.find ('.pic-link .img') .attr ('data-src')
'price':float (item.find ('.price'). Text () [2:])
'deal':item.find ('.deal-cnt') .text () [:-3]
'title':item.find ('.title') .text ()
'shop':item.find ('.shop') .text ()
'location':item.find ('.location') .text ()
'keywords':KEYWORDS
}
Save_to_mongo (product=product)
# Save to mongoDB
Def save_to_mongo (product):
Try:
If DB [Mongo _ TABLE] .insert (product):
Print ('saved successfully', product)
Except Exception:
Print ('failed to save')
# Central scheduling
Def main ():
Try:
Total = search ()
Total = int (re.compile ('. *? (d +). *?) .search (total) .group (1))
For i in range (2 focus totalizer 1):
Next_page (I)
Except Exception as e:
Print ('exception')
Finally:
Browser.close ()
If _ _ name__=='__main__':
Main ()
7. Running result
