End of an Era: Travel SEO

on in AJAX and Fetching Data
Last modified on

I’ll just leave this here for reference purposes and I won’t explain any code, as it’s obsolete and I’m not using it anyway. I also won’t go into any details on how to install Node.js or any of the other dependencies.

The main idea was to scrape the content and import it into WordPress as custom posts: hotels, venues, destinations, regions, countries. While I had it set up as I wanted and it was working properly, with AJAX requests and database caching, I had to stop due to lack of time and motivation. I learned a bit of Node.js in the process, so here’s my findings.

I was aiming for special offers and featured holidays, so I needed a way to scrape dynamic content and search results.

As a quick note, I know that the code below is pretty basic, but it might help someone in search of a quick solution.

Here’s how I scraped Booking.com hotels and extracted the hotel description:

var express = require('express');
var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');
var app = express();

app.get('/accommodation/', async function (req, res) {
    var accommodationArray = [
        'https://www.booking.com/hotel/es/trianflor.en-gb.html',
        'https://www.booking.com/hotel/es/apartamentos-doramar.en-gb.html',
        'https://www.booking.com/hotel/es/oasis-club.en-gb.html',
    ],
    url;
    var accommodation = [];

    for (let i = 0; i < accommodationArray.length; i++) {
        url = accommodationArray[i];

        let {err, resp, body} = await makeRequest(url);

        if (!err) {
            var $ = cheerio.load(body);

            let summary = $('#hotel_main_content #summary').text();
            console.log(url);

            accommodation.push([accommodationArray[i], summary]);
        }
    }

    res.send(accommodation);
});

function makeRequest(url) {
    return new Promise((resolve, reject) => {
        request(url, function (err, resp, body) {
            resolve({err, resp, body});
        });
    })
}

app.listen('8081');
console.log('Magic happens on port 8081');
exports = module.exports = app;

And here’s how I scraped more information based on a URL parameter:

var express = require('express');
var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');
var app = express();

function sanitize_string(str, keepCommas) {
    // Remove all single quotes
    str = str.replace(/'/g, '');

    // Remove all double quotes
    str = str.replace(/"/g, '');

    if (keepCommas === false) {
        // Remove all commas
        str = str.replace(/,/g, '');
    }

    // Remove the last comma and any whitespace after it
    str = str.replace(/,\s*$/, '');

    // Trim
    str = str.trim();

    return str;
}

app.get('/aod/:uri', async function (req, res) {
    var accommodationArray = [
        req.params.uri
    ],
    url;
    var accommodation = [];

    for (let i = 0; i < accommodationArray.length; i++) {
        url = accommodationArray[i];

        let {err, resp, body} = await makeRequest(url);

        if (!err) {
            var $ = cheerio.load(body);

            var regex;

            let summary = $('#hotel_main_content #summary').text().trim();
            //let address = $('.hp_address_subtitle').text();

            // Get coordinates
            regex = /(b_map_center_latitude.*)/g;
            let latitude = regex.exec(body);
            latitude = latitude[1];
            latitude = latitude.replace('b_map_center_latitude =', '');
            latitude = latitude.replace(';', '');
            latitude = sanitize_string(latitude, false);

            regex = /(b_map_center_longitude.*)/g;
            let longitude = regex.exec(body);
            longitude = longitude[1];
            longitude = longitude.replace('b_map_center_longitude =', '');
            longitude = longitude.replace(';', '');
            longitude = sanitize_string(longitude, false);

            // Get hotel ID
            regex = /(b_hotel_id.*)/g;
            let hotelId = regex.exec(body);
            hotelId = hotelId[1];
            hotelId = hotelId.replace('b_hotel_id =', '');
            hotelId = hotelId.replace('b_hotel_id:', '');
            hotelId = sanitize_string(hotelId, false);

            // Get country
            regex = /("addressCountry.*)/g;
            let country = regex.exec(body);
            country = country[1];
            country = country.replace('addressCountry" :', '');
            country = sanitize_string(country, false);

            // Get region
            regex = /("addressRegion.*)/g;
            let region = regex.exec(body);
            region = region[1];
            region = region.replace('addressRegion" :', '');
            region = sanitize_string(region, false);

            // Get address
            regex = /("streetAddress.*)/g;
            let address = regex.exec(body);
            address = address[1];
            address = address.replace('streetAddress" :', '');
            address = sanitize_string(address, true);

            // Get description
            regex = /("description" :.*)/g;
            let description = regex.exec(body);
            description = description[1];
            description = description.replace('description" :', '');
            description = sanitize_string(description, true);

            // Get photo
            regex = /(large_url.*)/g;
            let photo = regex.exec(body);
            photo = photo[1];
            photo = photo.replace('large_url:', '');
            photo = sanitize_string(photo, false);

            accommodation.push([accommodationArray[i], summary, address, country, region, description, photo, latitude, longitude, hotelId]);
        }
    }

    res.send(accommodation);
});

function makeRequest(url) {
    return new Promise((resolve, reject) => {
        request(url, function (err, resp, body) {
            resolve({err, resp, body});
        });
    })
}

app.listen('8081');
console.log('Magic happens on port 8081');
exports = module.exports = app;

And, finally, here’s a more specific scraping script for specific travel sites with different DOM structures:

var express = require('express');
var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');
var app = express();

function extract_details() {
    //
}

app.get('/scrape/:id', function (req, res) {
    if (req.params.id == 1) {
        url = 'https://www.example.com/family-holidays/';

        request(url, function (error, response, html) {
            if (!error) {
                var $ = cheerio.load(html),
                    offers = [];

                let offer = $('.cms-module-section');

                for (let i = 0; i < offer.length; i++) {
                    let source = 'Example Travel';
                    let sourceUri = 'https://www.example.com';
                    let title = $(offer[i]).find('h3.cms-txt1').text().trim();
                    let price = $(offer[i]).find('.price').text().trim();
                    let bb = $(offer[i]).find('p.cms-rating + p').text().trim();
                    let dateDuration = $(offer[i]).find('p.cms-rating + p + p').text().trim();
                    let origin = $(offer[i]).find('p.cms-rating + p + p + p').text().trim();
                    let uri = sourceUri + $(offer[i]).find('.cms-box-main a').attr('href');
                    let imageUri = sourceUri + $(offer[i]).find('.offer-img').attr('src');
                    let resort = $(offer[i]).find('h2.cms-nowrap a').text().trim();

                    imageUri = decodeURIComponent(imageUri);
                    imageUri = imageUri.replace('https://www.example.com/img.evolve?u=', '');
                    imageUri = imageUri.replace('https://www.example.comhttp', 'http');

                    dateDuration = dateDuration.replace(' ', ' ');
                    dateDuration = dateDuration.trim();
                    dateDuration = dateDuration.split('for');

                    if (title.length) {
                        offers.push([source, title, bb, dateDuration[0], dateDuration[1], origin, price, imageUri, uri, resort]);
                    }
                }

                res.send(offers);
            }
        });
    } else if (req.params.id == 2) {
        url = 'https://www.example.com/sun-holidays/sun-holidays-2019/';

        request(url, function (error, response, html) {
            if (!error) {
                var $ = cheerio.load(html),
                    offers = [];

                let offer = $('.cms-module');

                for (let i = 0; i < offer.length; i++) {
                    let source = 'Example Travel';
                    let sourceUri = 'https://www.example.com';
                    let title = $(offer[i]).find('.cms-module-main h3').text().trim();
                    let price = $(offer[i]).find('.price').text().trim();
                    let bb = '';
                    let dateDuration = $(offer[i]).find('.cms-module-main .theme-box-content .content')
                        .clone()    // clone the element
                        .children() // select all the children
                        .remove()   // remove all the children
                        .end()      // again go back to selected element
                        .text()     // get the text of element
                        .trim();
                    let uri = sourceUri + $(offer[i]).find('.cms-module-view a').attr('href');
                    let imageUri = $(offer[i]).find('.cms-module-main > img').attr('src');
                    let resort = $(offer[i]).find('.cms-module-main .theme-box-content .content h3 + div').text().trim();

                    imageUri = decodeURI(imageUri);

                    dateDuration = dateDuration.split("\n");
                    newDate = dateDuration[0];
                    duration = dateDuration[0].split('for');

                    origin = dateDuration[1];

                    if (title.length) {
                        offers.push([source, title, bb, duration[0].trim(), duration[1].trim(), origin, price, imageUri, uri, resort]);
                    }
                }

                res.send(offers);
            }
        });
    } else if (req.params.id == 3) {
        url = 'https://www.example.com/special-offers/';

        request(url, function (error, response, html) {
            if (!error) {
                var $ = cheerio.load(html),
                    offers = [];

                let offer = $('.cms-module');

                for (let i = 0; i < offer.length; i++) {
                    let source = 'Example Travel';
                    let title = $(offer[i]).find('.cms-box-info h3').text().trim();
                    let price = $(offer[i]).find('.cms-price').text().trim();
                    let uri = $(offer[i]).find('.cms-box-bottom a').attr('href');

                    let info = $(offer[i]).find('.cms-box-info > p.cms-no-wrap').text().trim();

                    // todo // change HTML structure
                    let bb = $(offer[i]).find('p.cms-rating + p').text().trim();
                    let dateDuration = $(offer[i]).find('p.cms-rating + p + p').text().trim();
                    let origin = $(offer[i]).find('p.cms-rating + p + p + p').text().trim();
                    let uri = sourceUri + $(offer[i]).find('.cms-box-main a').attr('href');
                    let imageUri = sourceUri + $(offer[i]).find('.offer-img').attr('src');
                    let resort = $(offer[i]).find('h2.cms-nowrap a').text().trim();

                    imageUri = decodeURIComponent(imageUri);
                    imageUri = imageUri.replace('https://www.example.com/img.evolve?u=', '');
                    imageUri = imageUri.replace('https://www.example.comhttp', 'http');

                    dateDuration = dateDuration.replace(' ', ' ');
                    dateDuration = dateDuration.trim();
                    dateDuration = dateDuration.split('for');

                    if (title.length) {
                        offers.push([title, price, uri]);
                    }
                }

                res.send(offers);
            }
        });
    } else if (req.params.id == 4) {
        url = 'https://www.example.com/winter-sun';

        request(url, function (error, response, html) {
            if (!error) {
                var $ = cheerio.load(html),
                    offers = [];

                let offer = $('.offersList ul li');

                for (let i = 0; i < offer.length; i++) {
                    let source = 'Example Travel';
                    let sourceUri = 'https://www.example.com';
                    let title = $(offer[i]).find('h2').text().trim();
                    let bb = '';
                    let price = $(offer[i]).find('.price').text().trim();
                    let uri = url + '/' + $(offer[i]).find('a.inner').attr('href');
                    let holidayDate = $(offer[i]).find('.badgeposition2').text().trim();
                    let duration = $(offer[i]).find('.infoList li').text().trim();
                    let origin = $(offer[i]).find('.badgeposition1.type1').text().trim();
                    let imageUri = sourceUri + '/' + $(offer[i]).find('.image img').attr('src');
                    let resort = '';

                    if (title.indexOf('-') > -1) {
                        title = title.split('-');
                        resort = title[0].trim();
                        title = title[1].trim();
                    }

                    if (price.indexOf('pp') > -1) {
                        price = price.split('pp');
                        bb = price[1].trim();
                        price = price[0].trim();
                    } else if (price.indexOf('per') > -1) {
                        price = price.split('per family');
                        bb = price[1].trim();
                        price = price[0].trim();
                    }
                    if (bb.indexOf('(') > -1 || bb.indexOf('based') > -1) {
                        bb = '';
                    }

                    holidayDate = holidayDate.replace('Book by', '').trim();
                    origin = origin.replace('From', '').trim();

                    if (title.length) {
                        offers.push([source, title, bb, holidayDate, duration, origin, price, imageUri, uri, resort]);
                    }
                }

                res.send(offers);
            }
        });
    } else if (req.params.id == 5) {
        url = 'https://www.example.com/cheap-holidays/lanzarote/7-Night-holidays-to-lanzarote/';

        request(url, function (error, response, html) {
            if (!error) {
                var $ = cheerio.load(html),
                    offers = [];

                let offer = $('.offers_iterator .flight_click_link');

                for (let i = 0; i < offer.length; i++) {
                    let source = 'Example Travel';
                    let sourceUri = 'https://www.example.com';
                    let title = $(offer[i]).find('div.where span').text().trim();
                    let bb = '';
                    let price = $(offer[i]).find('div.from strong').text().trim();
                    let uri = sourceUri + $(offer[i]).find('div.prices_from a').attr('href');
                    let holidayDate = $(offer[i]).find('div.from')
                        .clone()    // clone the element
                        .children() // select all the children
                        .remove()   // remove all the children
                        .end()      // again go back to selected element
                        .text()     // get the text of element
                        .trim();
                    let duration = $(offer[i]).find('div.howlong').text().trim();
                    let imageUri = $(offer[i]).find('img.dest_image').attr('src');
                    let resort = $(offer[i]).find('div.where')
                        .clone()    // clone the element
                        .children() // select all the children
                        .remove()   // remove all the children
                        .end()      // again go back to selected element
                        .text()     // get the text of element
                        .trim();
                    let origin = '';

                    holidayDate = holidayDate.replace('from', '').trim();
                    duration = duration.split('from');
                    origin = duration[1].trim();
                    duration = duration[0].trim();


                    if (title.length) {
                        offers.push([source, title, bb, holidayDate, duration, origin, price, imageUri, uri, resort]);
                    }
                }

                res.send(offers);
            }
        });
    }
});

app.listen('8082');
console.log('Magic happens on port 8082');
exports = module.exports = app;

Finally, here’s my package.json file:

{
  "name": "node-web-scrape",
  "version": "0.0.1",
  "description": "Scraper.",
  "main": "server.js",
  "author": "Me",
  "dependencies": {
    "express": "latest",
    "request": "latest",
    "cheerio": "latest"
  }
}

Related posts

JavaScript

April 23, 2023

Check DNS records in bulk using JavaScript

JavaScript Search Suggestions

September 8, 2023

A simple JavaScript suggestion autocomplete app

JavaScript

May 27, 2022

How to get content from another website using JavaScript