Wsrtjtyk

Working on scraping TV episodes from IMDb (Breaking Bad in the example below). The problem is when implementing the for loop, only the first iteration of j is returned.

My assumption is the return statement is exiting the loop but I'm unsure how to fix the problem.

const fetch = require('node-fetch');

const cheerio = require('cheerio');



const searchUrl = 'https://www.imdb.com/find?s=tt&ttype=tv&ref_=fn_tv&q=';

const movieUrl = 'https://www.imdb.com/title/';



async function getEpisodes(searchTerm) {



  //const imdbID = await getID(searchTerm);

  //const numSeasons = await getSeasons(imdbID);



  const imdbID = 'tt0903747';

  const numSeasons = 5;

  const episodes = ;



  for (let j = 1; j <= numSeasons; j++) {

    return fetch(`${movieUrl}${imdbID}/episodes?season=${j}`)

      .then(response => response.text())

      .then(body => {

        const $ = cheerio.load(body);



        $('div[itemProp="episodes"]').each(function (i, element) {

          const airdate = $(element).find('.airdate').text().trim();

          const episodeTitle = $(element).find('a[itemProp="name"]').text().trim();

          const votes = $(element).find('.ipl-rating-star__total-votes').text().trim().match(/(([^)]+))/)[1];

          const rating = $(element).find('.ipl-rating-star ').find('.ipl-rating-star__rating').text().trim().slice(0, 3);



          episode = {

            season: j,

            episodeTitle,

            airdate,

            votes,

            rating

          };

          episodes.push(episode);

        }); 

        return episodes; //Only season 1 is returned.

      }); 

  }

}

Let's rewrite the function using async await style. This way we make sure we fire fetch numSeasons times, await all of them, and process them one by one.

async function processResponse(response, season) {

    const body = await response.text();

    const $ = cheerio.load(body);



    let episodes = ;

    $('div[itemProp="episodes"]').each(function (i, element) {

        const airdate = $(element).find('.airdate').text().trim();

        const episodeTitle = $(element).find('a[itemProp="name"]').text().trim();

        const votes = $(element).find('.ipl-rating-star__total-votes').text().trim().match(/(([^)]+))/)[1];

        const rating = $(element).find('.ipl-rating-star ').find('.ipl-rating-star__rating').text().trim().slice(0, 3);



        episode = {

            season,

            episodeTitle,

            airdate,

            votes,

            rating

        };



        episodes.push(episode);

    });



    return episodes;

}



async function getEpisodes(searchTerm) {



    //const imdbID = await getID(searchTerm);

    //const numSeasons = await getSeasons(imdbID);



    const imdbID = 'tt0903747';

    const numSeasons = 5;



    let promises = ;



    for (let j = 1; j <= numSeasons; j++) {

        promises.push(fetch(`${movieUrl}${imdbID}/episodes?season=${j}`));

    }



    const responses = await Promise.all(promises);

    return responses.reduce((accumulator, response, index) => {

        return accumulator.concat(await processResponse(response, index + 1));

    }, );

}