Блог

Я пытаюсь получить значение из websraping, однако я не могу построить правильный класс для значения

Я хочу узнать дату со страницы:

Мой текущий код:

 class GameData:
    def __init__(self):
        self.score = []
        self.date = []
        self.time = []
        self.country = []
        self.league = []
        self.game = []
        self.home_odds = []
        self.draw_odds = []
        self.away_odds = []

    def append(self, score):
        pass


def get_urls(browser, landing_page):
    browser.get(landing_page)
    urls = [i.get_attribute('href') for i in
            browser.find_elements_by_css_selector(
                '.next-games-date > a:nth-child(1), .next-games-date > a:nth-child(n 3)')]

    return urls


def parse_data(html):
    df = pd.read_html(html, header=0)[0]
    html = browser.page_source
    soup = bs(html, "lxml")
    cont = soup.find('div', {'id': 'wrap'})
    content = cont.find('div', {'id': 'col-content'})
    content = content.find('table', {'class': 'table-main'}, {'id': 'table-matches'})
    main = content.find('th', {'class': 'first2 tl'})

    if main is None:
        return None

    count = main.findAll('a')
    country = count[0].text
    game_data = GameData()
    game_date = cont.find('h1', {'class': 'cms'})
    game_date = game_date[1].text


    for row in df.itertuples():
        if not isinstance(row[1], str):
            continue
        elif ':' not in row[1]:
            country = row[1].split('»')[0]
            continue
        game_time = row[1]
        score = row[3] if row[3] else nan

        game_data.date.append(game_date)
        game_data.time.append(game_time)
        game_data.country.append(country)
        game_data.league.append(count[1].text)
        game_data.game.append(row[2])
        game_data.score.append(score)
        game_data.home_odds.append(row[4])
        game_data.draw_odds.append(row[5])
        game_data.away_odds.append(row[6])

    return game_data


if __name__ == '__main__':

    start_url = "https://www.oddsportal.com/matches/soccer/"
    urls = []
    browser = webdriver.Chrome()
    results = None
    urls = get_urls(browser, start_url)
    urls.insert(0, start_url)

    for number, url in enumerate(urls):
        if number > 0:
            browser.get(url)
        html = browser.page_source
        game_data = parse_data(html)

        if game_data is None:
            continue

        result = pd.DataFrame(game_data.__dict__)

        if results is None:
            results = result
        else:
            results = results.append(result, ignore_index=True)
 

Ошибка:

 Traceback (most recent call last):
  File "C:/Users/harsh/AppData/Roaming/JetBrains/PyCharmCE2021.1/scratches/scratch_16.py", line 101, in <module>
    game_data = parse_data(html)
  File "C:/Users/harsh/AppData/Roaming/JetBrains/PyCharmCE2021.1/scratches/scratch_16.py", line 63, in parse_data
    game_date = game_date[1].text
TypeError: 'NoneType' object is not subscriptable
 

Я пытаюсь xpath найти объект:

 //*[@id="col-content"]/h1

full xpath

/html/body/div[1]/div/div[2]/div[6]/div[1]/div/div[1]/div[2]/div[1]/h1
 

Как мне получить game_date значение.

Вы можете извлечь из URL-адреса, указанного в первом элементе с классом bold .

 import datetime

# other code
game_date = datetime.datetime.strptime(soup.select_one('.bold')['href'].split('/')[-2],'%Y%m%d').date()