songs-lyrics-generator/scrapper.py

import random
import pandas as pd
from bs4 import BeautifulSoup
import requests
import os
import time
from ScrapThread import ScrapThread
from proxy_handling import proxies_validation


def connect(url, proxies_list):
    headers = {
        'User-Agent': "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.30 (KHTML, like Gecko) "
                      "Chrome/12.0.742.112 Safari/534.30"}
    main_page = None
    while True:
        try:
            main_page = requests.get(url, headers=headers) #, proxies={'http': random.choice(proxies_list),
                                                           #         'https': random.choice(proxies_list)}, timeout=5.0)
            break
        except:
            continue
    if main_page.status_code != 200:
        raise Exception("Access Denied!")
    return main_page


def get_lyricsAzlyrics(url, proxies_list):
    main_page_text = BeautifulSoup(connect(url, proxies_list).text, "html.parser")
    lyrics = str()
    for goods in main_page_text.find_all("div", {"class": None}):
        if len(goods.text) == 0:
            pass
        lyrics = lyrics + goods.text
    return lyrics


def scrapAzlyrics(url, selected_albums, time_stamp, proxies_list):
    main_page_text = BeautifulSoup(connect(url, proxies_list).text, "html.parser")
    albums = main_page_text.find_all(class_="album")
    if not albums:
        raise Exception("Access Denied!")
    scraped_lyrics = pd.DataFrame(columns=['Title', 'Lyrics'])
    for alb in albums:
        album_title = alb.find("b").text
        album_title = album_title[1:-1]
        print("Checking album: ", album_title)
        if album_title in selected_albums:
            next_song = alb.find_next(class_="listalbum-item")
            if next_song.find(class_="comment"):
                next_song = next_song.find_next(class_=["listalbum-item", "album"])
            while True:
                if not next_song.find(class_="comment"):
                    title = next_song.find("a").text
                    lyrics = get_lyricsAzlyrics(("https://www.azlyrics.com/" + next_song.find("a")['href']),
                                                proxies_list)
                    df = pd.DataFrame({
                        "Title": [title],
                        "Lyrics": [lyrics]
                    })
                    scraped_lyrics = pd.concat([scraped_lyrics, df], ignore_index=True)
                    print("Downloaded song: ", title)
                    next_song = next_song.find_next(class_=["listalbum-item", "album"])
                    if next_song['class'] != ['listalbum-item']:
                        break
                else:
                    next_song = next_song.find_next(class_=["listalbum-item", "album"])
                    if next_song['class'] != ['listalbum-item']:
                        break
                time.sleep(time_stamp)
    return scraped_lyrics


def get_lyricsTekstowo(url, proxies_list):
    main_page_text = BeautifulSoup(connect(url, proxies_list).text, "html.parser")
    lyrics = str()
    for goods in main_page_text.find_all("div", {"class": "inner-text"}):
        if len(goods.text) == 0:
            pass
        if goods.parent.attrs['class'] == ['song-text']:
            lyrics = lyrics + goods.text
    return lyrics


def scrapTekstowo(url, proxies_list):
    main_page_text = BeautifulSoup(connect(url, proxies_list).text, "html.parser")
    song_list = main_page_text.find(class_='ranking-lista')
    songs = song_list.find_all(class_="box-przeboje")
    scraped_lyrics = pd.DataFrame(columns=['Title', 'Lyrics'])
    if not songs:
        raise Exception("Access Denied!")
    for song in songs:
        ref = song.find("a")
        title = ref.text.split('-')[1][1:]
        lyrics = get_lyricsTekstowo('https://www.tekstowo.pl' + ref['href'], proxies_list)
        df = pd.DataFrame({
            "Title": [title],
            "Lyrics": [lyrics]
        })
        scraped_lyrics = pd.concat([scraped_lyrics, df], ignore_index=True)
        print("Downloaded song: ", title)
    return scraped_lyrics


def check_pagination(url, proxies_list):
    main_page_text = BeautifulSoup(connect(url, proxies_list).text, "html.parser")
    pages = main_page_text.find_all(class_='page-link')
    page_number = 0
    for page in pages:
        if page.text != 'Następna >>' and page.text != '<< Poprzednia':
            page_number = page_number + 1
    return page_number / 2


def do_threading(url, selected_albums, time_stamp, proxies_list):
    threads = []
    thread_number = 0
    if url.split('/')[2] == 'www.azlyrics.com':
        thread_number = len(selected_albums)
    if url.split('/')[2] == 'www.tekstowo.pl':
        thread_number = int(check_pagination(url, proxies_list))
    df = pd.DataFrame(columns=['Title', 'Lyrics'])
    for i in range(thread_number):
        t = None
        if url.split('/')[2] == 'www.azlyrics.com':
            t = ScrapThread(target=scrapAzlyrics, args=(url, [selected_albums[i]], time_stamp, proxies_list))
        if url.split('/')[2] == 'www.tekstowo.pl':
            newUrl = url[:-5] + ',strona,' + str(i + 1) + '.html'
            t = ScrapThread(target=scrapTekstowo, args=(newUrl, proxies_list))
        t.daemon = True
        threads.append(t)
    for i in range(thread_number):
        threads[i].start()
    for i in range(thread_number):
        df = pd.concat([df, threads[i].join()], ignore_index=True)
    return df


def scrap_data(url, selected_albums, time_stamp):
    # proxies_list = proxies_validation()
    proxies_list = []
    df = do_threading(url, selected_albums, time_stamp, proxies_list)
    path = os.path.dirname(os.path.abspath(__file__))
    path = os.path.join(path, "Data")
    if url.split('/')[2] == 'www.azlyrics.com':
        filename = url.split('/')[4][:-5] + '.csv'
        saving = os.path.join(path, filename)
        df.to_csv(saving)
    if url.split('/')[2] == 'www.tekstowo.pl':
        filename = url.split(',')[1][:-5] + '.csv'
        saving = os.path.join(path, filename)
        df.to_csv(saving)
    # os.remove("valid_proxy_list")