#selenium #web-scraping #screen-scraping
Вопрос:
Селеновый код
from bs4 import BeautifulSoup as soup
from urllib.request import urlopen as uReq
import csv
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.options import Options
import os
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
# constant selenium options
opts= Options()
opts.add_argument("--headless")
opts.add_argument('--ignore-certificate-errors')
opts.add_argument('--incognito')
opts.page_load_strategy = 'normal'
# first I want a list of all the seasons plus the link of those seasons
page = "https://www.iplt20.com/teams"
uClient = uReq(page)
page_html = uClient.read()
uClient.close()
page_soup = soup(page_html,"html.parser")
teams =[]
ul = page_soup.find("ul",{"class":"content-grid"})
#print(ul)
lis = ul.findAll("li",{"class":"team-card-grid__item"})
for li in lis:
l=[]
l.append(li.find("h3",{"class":"card__title"}).text)
try:
l.append(li.find("div",{"class":"team-card__wins"}).text.strip())
except:
l.append('')
#print(l[0]," doesn't win")
l.append("https://www.iplt20.com" li.find("a",{"class":"card"})["href"] "/squad")
#print(l)
teams.append(l)
print(teams)
for team in teams:
team_page = team[2]
print(team_page)
driver = webdriver.Chrome(options=opts)
driver.get(team_page)
print(driver.title)
team_page_soup = driver.page_source
team_page_beautify = soup(team_page_soup,features="lxml")
#print(team_page_beautify)
driver.save_screenshot(team[0] ".png")
player_list = driver.find_element_by_class_name("playersList")
players = player_list.find_elements_by_tag_name("li")
print(players)
#here I am getting an empty players list
for player in players:
print(player.text)
break
Мне нужно очистить несколько страниц динамическим способом, но я не могу получить элементы списка ul с помощью списка игроков класса.
Каковы другие способы сделать то же самое?
Кроме того, будет здорово, если вы предоставите несколько кратких и точных ссылок, из которых я смогу узнать больше.
Комментарии:
1. Может быть, попробуйте добавить
opts.add_argument("--start-maximized"), чтобы вы могли видеть все в окне просмотра?2. К вашему сведению, это скребок (и скребок, скребок, скребок), а не лом. Сдавать на металлолом — значит выбрасывать мусор, как мусор.
Ответ №1:
Вы можете получить команду из api. Просто вопрос в том, чтобы ввести идентификатор команды.
import requests
import pandas as pd
rows = []
for i in [1, 3, 4, 5, 6, 8, 9, 62]:
squad_api = 'https://cricketapi.platform.iplt20.com/tournaments/22399/squads/%s?matchTypes=ALL' %i
jsonData = requests.get(squad_api).json()
teamData = jsonData['ALL']['team']
teamData = { 'team_' your_key: teamData[your_key] for your_key in teamData.keys() }
playerData = jsonData['ALL']['players']
print(teamData['team_fullName'])
for player in playerData:
player = { 'player_' your_key: player[your_key] for your_key in player.keys() }
print('t',player['player_fullName'])
row = teamData.copy()
row.update(player)
rows.append(row)
df = pd.DataFrame(rows)
Выход:
print(df.head(30).to_string())
team_id team_fullName team_shortName team_abbreviation team_type team_primaryColor team_secondaryColor player_id player_fullName player_shortName player_nationality player_dateOfBirth player_rightArmedBowl player_rightHandedBat player_bowlingStyle
0 1 Chennai Super Kings Super Kings CSK m FDB913 0081E5 100 Ambati Rayudu A Rayudu Indian 1985-09-23 True True OFF_SPIN
1 1 Chennai Super Kings Super Kings CSK m FDB913 0081E5 4944 KM Asif KM Asif Indian 1993-07-24 True True MEDIUM_SEAM
2 1 Chennai Super Kings Super Kings CSK m FDB913 0081E5 25 Dwayne Bravo DJ Bravo West Indian 1983-10-07 True True MEDIUM_SEAM
3 1 Chennai Super Kings Super Kings CSK m FDB913 0081E5 140 Deepak Chahar D Chahar Indian 1992-08-07 True True MEDIUM_SEAM
4 1 Chennai Super Kings Super Kings CSK m FDB913 0081E5 24 Faf du Plessis F du Plessis South African 1984-07-13 True True LEG_SPIN
5 1 Chennai Super Kings Super Kings CSK m FDB913 0081E5 898 Imran Tahir I Tahir South African 1979-03-27 True True LEG_SPIN
6 1 Chennai Super Kings Super Kings CSK m FDB913 0081E5 4942 Narayan Jagadeesan N Jagadeesan Indian 1995-12-24 True True NaN
7 1 Chennai Super Kings Super Kings CSK m FDB913 0081E5 1118 Karn Sharma K Sharma Indian 1987-10-23 True False LEG_SPIN
8 1 Chennai Super Kings Super Kings CSK m FDB913 0081E5 3746 Lungi Ngidi L Ngidi South African 1996-03-29 True True FAST_SEAM
9 1 Chennai Super Kings Super Kings CSK m FDB913 0081E5 1903 Mitchell Santner M Santner New Zealander 1992-02-05 False False ORTHODOX
10 1 Chennai Super Kings Super Kings CSK m FDB913 0081E5 1 MS Dhoni MS Dhoni Indian 1981-07-07 True True MEDIUM_SEAM
11 1 Chennai Super Kings Super Kings CSK m FDB913 0081E5 7123 R Sai Kishore RS Kishore Indian 1996-11-06 False False ORTHODOX
12 1 Chennai Super Kings Super Kings CSK m FDB913 0081E5 9 Ravindra Jadeja R Jadeja Indian 1988-12-06 False False ORTHODOX
13 1 Chennai Super Kings Super Kings CSK m FDB913 0081E5 127 Robin Uthappa R Uthappa Indian 1985-11-11 True True NaN
14 1 Chennai Super Kings Super Kings CSK m FDB913 0081E5 5443 Ruturaj Gaikwad R Gaikwad Indian 1997-01-31 True True OFF_SPIN
15 1 Chennai Super Kings Super Kings CSK m FDB913 0081E5 2939 Sam Curran S Curran English 1998-06-03 False False MEDIUM_SEAM
16 1 Chennai Super Kings Super Kings CSK m FDB913 0081E5 1745 Shardul Thakur S Thakur Indian 1991-10-16 True True MEDIUM_SEAM
17 1 Chennai Super Kings Super Kings CSK m FDB913 0081E5 14 Suresh Raina S Raina Indian 1986-11-27 True False OFF_SPIN
18 1 Chennai Super Kings Super Kings CSK m FDB913 0081E5 1735 Moeen Ali M Ali English 1987-06-18 True False OFF_SPIN
19 1 Chennai Super Kings Super Kings CSK m FDB913 0081E5 3834 Krishnappa Gowtham K Gowtham Indian 1988-10-20 True True OFF_SPIN
20 1 Chennai Super Kings Super Kings CSK m FDB913 0081E5 156 Cheteshwar Pujara C Pujara Indian 1988-01-25 True True LEG_SPIN
21 1 Chennai Super Kings Super Kings CSK m FDB913 0081E5 9474 Harishankar Reddy H Reddy Indian 1998-06-02 True True FAST_SEAM
22 1 Chennai Super Kings Super Kings CSK m FDB913 0081E5 7111 C Hari Nishaanth C Hari Nishaanth Indian 1996-08-16 True False OFF_SPIN
23 1 Chennai Super Kings Super Kings CSK m FDB913 0081E5 7105 Bhagath Varma B Varma Indian 1998-09-21 True True OFF_SPIN
24 1 Chennai Super Kings Super Kings CSK m FDB913 0081E5 937 Jason Behrendorff J Behrendorff Australian 1990-04-20 False True FAST_SEAM
25 3 Delhi Capitals Delhi DC m 004C93 F83430 135 Ajinkya Rahane A Rahane Indian 1988-06-06 True True MEDIUM_SEAM
26 3 Delhi Capitals Delhi DC m 004C93 F83430 30 Amit Mishra A Mishra Indian 1982-11-24 True True LEG_SPIN
27 3 Delhi Capitals Delhi DC m 004C93 F83430 5433 Anrich Nortje A Nortje South African 1993-11-16 True True FAST_SEAM
28 3 Delhi Capitals Delhi DC m 004C93 F83430 1561 Avesh Khan A Khan Indian 1996-12-13 True True FAST_SEAM
29 3 Delhi Capitals Delhi DC m 004C93 F83430 1113 Axar Patel A Patel Indian 1994-01-20 False False ORTHODOX