Блог

В настоящее время я создаю чат-бота telegram, который просматривает стихи. У меня возникают проблемы с «очисткой» моего вывода веб-скребка, особенно когда я хочу получить аудиозапись из www.poetryarchive.org текст.

Это мой текущий код:

 # import relevant libraries for web scraping and asynchronous input/output
import asyncio
import telepot
import telepot.aio
from telepot.aio.loop import MessageLoop
from pprint import pprint
from bs4 import BeautifulSoup
import requests

# Telegram bot API token (and its configuration to aysnc version)
TOKEN = '<INSERT API TOKEN HERE>'
bot = telepot.aio.Bot(TOKEN)

# This function will be modified to accommodate our web scraper code afterwards
async def handle(msg):
    # Creating a global variable
    global chat_id
    # These variables will allow us to glance at the received messages - extract the messages' 'headline info'
    content_type, chat_type, chat_id = telepot.glance(msg)
    # Log variables
    print(content_type, chat_type, chat_id)
    pprint(msg)
    username = msg['chat']['first_name']
    if content_type == 'text':
        # map the message to the '/start' function
        if msg['text'] == '/start':
            await bot.sendMessage(chat_id, "Hi! I am PoeticScraper and I am able to scrape your desired poem, its credits, the poet's information, and the poem's glossary tags. Key in '/help' for more assistance or any keywords without the '/' to get the ball rolling!")
        # map the message to the '/help' function
        elif msg['text'] == '/help':
            await bot.sendMessage(chat_id, "I am here to help! For starters, you can type any keywords in and I will retrieve the most relevant poem for you! Alternatively, if you have specific poems you want to search, you can do so too! But please note the for poem titles with multiple words, please use '-' between them!")
        elif msg['text'] == '/more':
            await bot.sendMessage(chat_id, "Can't get enough of poems? Fret not! There are so many other poetry websites for you to browse: www.poetryfoundation.org amp; www.poets.org")
        elif msg['text'] != '/start' or '/help':
            text = msg['text']
            # it's better to strip and lower the input in order for the subsequent function to comprehend it
            text = text.strip()
            await getInformation(text.lower())
        # if all else fails...
        else:
            await bot.sendMessage(chat_id, "404 not found!")

# the variable 'headers' is required so that the target website perceives our web-scraper as a browser
headers = {'User-Agent':
           'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36'}

# defining a function - in charge of making the HTTP request, receiving an HTML response, and retrieving the poem
async def getInformation(text):
    # specify the url - all the URLs on this website follow this convention: 'https://poetryarchive.org/poem/<insert poem name>/'
    mainpage_url = 'https://poetryarchive.org/poem/'   text
    # query the website and return the html to the variable 'page'
    page = requests.get(mainpage_url, headers = headers)
    # parse the html using BeautifulSoup and store in variable `soup`
    soup = BeautifulSoup(page.text, 'html.parser')
    pprint(soup)
# this part is crucial to web-scraping the required information from the webpage
    try:
        # poet's details
        try:
            for poet in soup.find_all('h3', {'style': 'margin-top:0px;'}):
                for poet_name in poet.find_all('a'):
                    pprint(poet_name.get_text('href'))
                    await bot.sendMessage(chat_id, poet_name)
        except:
            await bot.sendMessage(chat_id, "The poet's bio is currently unavailable.")
        # credits/copyright info about the poem
        try:
            credits = soup.find('div', {'class': 'source-box bg-grey pa-boxed small-text'}).text
            await bot.sendMessage(chat_id, credits)
        except:
            await bot.sendMessage(chat_id, "No credits found!")
        # audio version of the poem
        try:
            for audio_recording in soup.find_all('div', {'class': 'pa-player-wrap'}):
                pprint(audio_recording.get('audiosrc'))
                await bot.sendMessage(chat_id, audio_recording)
        except:
            await bot.sendMessage(chat_id, "Sorry. The audio recording is currently unavailable.")
        # textual version of the poem
        try:
            poem = soup.find('div', {'class': 'poem-content'}).text
            await bot.sendMessage(chat_id, poem)
        except:
            await bot.sendMessage(chat_id, "Sorry. The poem is currently unavailable.")
        # genre types for this poem
        try:
            for meta_tags in soup.find_all('div', {'class': 'player-metas'}):
                for tag in meta_tags.find_all('a'):
                    pprint(tag.get_text('href'))
                    await bot.sendMessage(chat_id, "These are the glossary tags associated with this poem: ")
                    await bot.sendMessage(chat_id, tag)
        except:
            await bot.sendMessage(chat_id, "Sorry. The glossary tags for this poem are currently unavailable.")
    except:
        await bot.sendMessage(chat_id, "Oh no! Something went wrong... Please enter '/help' for more information.")

# Program startup
loop = asyncio.get_event_loop()
loop.create_task(MessageLoop(bot, handle).run_forever())
print('Listening ...')

# Keep the program running
loop.run_forever()
  

Это вывод телеграммы, который я получил для стихотворения «Upstream»:

 <div class="pa-player-wrap">
<div class="df-close"></div>
<div class="df-title"><h6>Upstream - Jean Binta Breeze</h6></div> <div audiosrc="https://poetryarchive.org/wp-content/uploads/poems/full/Upstream.mp3" class="pa-player for-poem-3216 paused">
<a class="pa-poem-play pa-player-control"></a>
<div class="pa-poem-progress">
<div class="timer current"></div>
<div class="pa-progress-slider">
<div class="pa-loading-bar"></div>
<div class="pa-progress-bar"></div>
</div>
<div class="timer full"></div>
</div>
<div class="pa-poem-volume" title="Volume">
<a class="pa-volume pa-player-control"></a>
<div class="pa-volume-slider-wrap">
<div class="pa-volume-slider">
<div class="pa-volume-bar">
<div class="pa-volume-handlebar" style="width: 100%;"></div>
</div>
</div>
</div>
</div>
<a class="close-df"><i class="fal fa-fw fa-times c-pink"></i></a>
<a class="close-eyes pa-player-control" title="Distraction free">
<div class="eye eye-open"></div>
<div class="eye eye-shut"></div>
</a>
<div class="pa-poem-add">
<div class="member-fav nonmember-fav" title="Become a member to add to a collection"></div> </div>
<a class="pa-poem-drag" title="Reorder collection"></a>
</div>
</div>
  

Что я могу сделать, чтобы я мог просто найти URL-адрес в теге? Я перепробовал так много методов, но это не сработало.

Следующее может сработать, если вы используете селекторы css для назначения audiosrc атрибута элемента с помощью class pa-player (часть многозначного класса)

 audio = soup.select_one('.pa-player[audiosrc]')
message_link = 'Audio link not found'

if audio:
    message_link = audio['audiosrc']