Wsrtjtyk

I need to create the dataset on news. I need to extract all the news of given news website which have ever been posted on that website. I have write this code

import requests     

from bs4 import BeautifulSoup    

import pandas    

import csv    

from datetime import datetime



records=



def cnbc(base_url):



    r = requests.get(base_url)    

    c = r.content    

    soup = BeautifulSoup(c,"html.parser")    

    Title=soup.find("h1","class":"title"}).text.replace("r","").replace("n","")

    content=' '

    for content_tag in soup.find_all("p"):

        content = content+content_tag.text.replace("r","").replace("n","")

    content= content[18:-458]

    Country ='United States'

    website='https://www.cnbc.com/' 

    comments='' 

    genre='Political'

    date= soup.find("time",{"class":"datestamp"}).text[35:-2].replace("r","").replace("n","")

    d = datetime.strptime(date, "%d %b %Y")

    date = d.strftime("%d-%m-%Y")

    records.append((Title,content,Country,website,comments,genre,date))



cnbc("https://www.cnbc.com/2018/11/02/here-are-the-three-things-pulling-down-the-stock-market-again.html")

but this is only allowing me to extract one news.

Can anyone tell me that how can I extract all the news url from the root directory of website.

This is python3 script and it is not flawless but I hope it can serve as starting point so you can achieve what you are trying to. I am not sure does this site from which you are trying to scrape data allows such operation so I will not place their web address for constants WEB_SITE_BASE_URL and WEB_SITE_REGION_URL. It is your choice what you are going to put there.

import requests     

from bs4 import BeautifulSoup     

from datetime import datetime



# https://www.xxxx.com"

WEB_SITE_BASE_URL= ""

# https://www.xxxx.com/?region=us

WEB_SITE_REGION_URL = ""



def get_categories(web_site_base_url):

    r = requests.get(web_site_base_url)    

    c = r.content    

    soup = BeautifulSoup(c,"html.parser")    

    spans = soup.find_all(attrs={"nav-menu-buttonText"})

    categories = [category.text for category in spans]

    return categories



def get_links(category_url):

    r = requests.get(category_url)

    c = r.content    

    soup = BeautifulSoup(c,"html.parser")

    links = [a.get('href') for a in soup.find_all('a', href=True)]

    filtered_links = list(set([k for k in links if '/2018/11/' in k]))

    return filtered_links



def news(link):

    r = requests.get(link)    

    c = r.content    

    soup = BeautifulSoup(c,"html.parser")    

    Title=soup.find("h1",{"class":"title"}).text.replace("r","").replace("n","")

    content=' '

    for content_tag in soup.find_all("p"):

        content = content+content_tag.text.replace("r","").replace("n","")

    content= content[18:-458]

    Country ='United States'

    website='WEB_SITE_BASE_URL' 

    comments=''

    date= soup.find("time",{"class":"datestamp"}).text[35:-2].replace("r","").replace("n","")

    d = datetime.strptime(date, "%d %b %Y")

    date = d.strftime("%d-%m-%Y")  

    spans = soup.find_all(attrs={"header_title last breadcrumb"})

    categories = [category.text for category in spans]

    genre = categories

    return(Title,content,Country,website,comments,genre,date)



categories = get_categories(WEB_SITE_REGION_URL)

list_of_link_lists = 

for category in categories:

    list_of_link_lists.append(get_links(WEB_SITE_BASE_URL + "/" + category.replace(" ", "20%")))

flat_link_list = list(set([item for sublist in list_of_link_lists for item in sublist]))

articles_list = 

for link in flat_link_list:

    try:

        articles_list.append(news(WEB_SITE_BASE_URL + link))

    except:

        print("Something was wrong")

    continue



print(articles_list)

There is a rough method to extract part of all news, the method showed as my code. First, extract all the news which div class is headline news_headline = soup.find_all("div",class_="headline"). Then check if element is what we want.

new = 

for div in news_headline:

    each = ()

    if div.a:

        each[0] = url + div.a.get("href")

        if div.a.text:

            # use split to remove t n blankspace

            each[1] = " ".join(div.a.text.split())

        else:

            each[1] = " ".join(div.a.get("title").split())

        new.append(each)

    else:

        continue

It is the full code but i wrote this as short as i can.

import requests     

from bs4 import BeautifulSoup    



def index(url="https://www.cnbc.com/world/"):

    with requests.Session() as se:

        se.encoding = "UTF-8"

        res = se.get(url)

        text = res.text

    soup = BeautifulSoup(text,"lxml")

    news_headline = soup.find_all("div",class_="headline")

    news_ = [(url + div.a.get("href"), " ".join(div.a.text.split()) if div.a.text else  "".join(div.a.get("title").split()) ) for div in news_headline if div.a]

    print(news_)



index()