Suppose you work for a digital marketing agency, and you're head of business development has asked you for some help with market research. They give you an Excel file with information on 100 businesses containing bits of information such as "Name of Company," "Business Phone Number," and "Business Address." As a first step, you want to plot each business on a map to begin looking for interesting patterns. The script below will show you how to capture geo data (i.e., latitude and longitude) from each address using Google Maps GeoCoder API.
The example below uses Google Maps API, but many services can provide geo-coded data. A few other providers I recommend are MapBox, OpenCage GeoCoder and even the US Census can provide interesting insights.
The NodeJS Script is written in ES6 and uses FakerJS to create fake address data. The class FakerUtils has a method users that will automatically create between 1 and 20 new company addresses. The idea behind the class is to simulate the company data you get in the form of an Excel spreadsheet. The script then sends the data to another custom class, GeoGoogleUtils, to collect the geo-coded data in the form of formattedAddress, latitude and longitude. Once an array of objects is returned, I then filter out any blank addresses. Again, I'm simulating the idea that some Excel data might either be invalid or outdated.
When it comes to filtering our empty data within an array of Promises, there are a few ways to do it. Therefore, I present three variations so that it's easier to figure out what technique works best.
Enjoy!
const NodeGeocoder = require('node-geocoder');
const faker = require('faker');
const config = {
api: {
google: { provider: 'google', apiKey: 'https://developers.google.com/maps/documentation/javascript/get-api-key' }
}
}
class FakerUtils {
/**
* Create a Schema of Fake data you want to test
* returns {object} - Schema
*/
static get schema() {
return {
id: '{{random.number}}',
company: '{{company.companyName}} {{company.companySuffix}}',
address: '{{address.streetAddress(true)}}',
fname: '{{name.firstName}}',
lname: '{{name.lastName}}',
phone: '{{phone.phoneNumber}}',
email: '{{internet.email}}'
}
}
/**
* Get a random number of users between 1 and 20 based on a Schema
* returns {array} - Random Array of Users based on a Schema
*/
static get users(){
return this.generateRandomData(this.schema, 1, 20)
}
/**
* Get a single user object based on a Schema
* returns {array} - A single user array based on a Schema
*/
static get user(){
return this.generateRandomData(this.schema, 1, 1)
}
/**
* Generate Random Data Based on a Specific Schema
* returns {array} - Array of objects based on a Schema
*/
// Generate random data based on the schema above
static generateRandomData = (schema, min, max) => {
max = max || min
return Array.from({ length: faker.random.number({ min, max }) }).map(() => Object.keys(schema).reduce((entity, key) => {
entity[key] = faker.fake(schema[key])
return entity
}, {}))
}
}
class GeoGoogleUtils {
get OPTIONS() {
return config.api.google;
}
constructor() {
this.geocoder = NodeGeocoder(this.OPTIONS);
}
/**
* Conditionally build an object from address, latitude, and longitude
* @param {object} - Deconstruct params from Google Geo Coder
* returns {array:Promise} - Array of Promises that contain new Object
*/
parser({ formattedAddress, latitude, longitude }) {
//console.log(`${formattedAddress}\n[${latitude},${longitude}]`)
return {
...(formattedAddress && { formattedAddress: (formattedAddress !== 'undefined') ? formattedAddress : '' }),
...(latitude && { latitude: (latitude !== 'undefined') ? latitude : '' }),
...(longitude && { longitude: (longitude !== 'undefined') ? longitude : '' })
}
}
/**
* Gather geo data for a single city names and zip code
* @param {string} - City name or zip code as String.
* returns {object} - Geocoder Object
*/
async search(string) {
const geoObject = await this.geocoder.geocode(string)
.catch(GeoCodeError => GeoCodeError);
return geoObject;
}
/**
* Gather geo data for an array of city names and zip codes in parallel
* @param {array} - Array of type string.
* returns {arrray:Promise} - Array of Promises that contain Geocoder Object
*/
async parallelSearch(array){
const cityPromises = array.map(async city => {
const response = await this.geocoder.geocode(city)
.catch(GeoCodeError => GeoCodeError);
let responseData = response.map(item => item)[0]
// If there's no data, return a blank {}
if(typeof responseData === 'undefined'){
return { }
} else {
//console.log( this.parser(responseData) );
let parsedData = this.parser(responseData)
return { address: parsedData.formattedAddress, lat: parsedData.latitude, lon: parsedData.longitude }
}
});
// Log them in sequence
// for (const cityPromise of cityPromises) {
// console.log(await cityPromise);
// }
return cityPromises
}
}
(async () => {
// A. Create a new Geo Cordinates Object
let googleGeoUtils = new GeoGoogleUtils();
// B. Create an array of Fake user addresses
let addressArr = FakerUtils.users;
console.log(`Random Array of Addresses: [${addressArr.map(item => "'" + item.address + "'")}]`)
// C. Reverse geo code all the addresses in parallel and find the ones that have a proper lat and lon
let data = googleGeoUtils.parallelSearch(addressArr)
// Option 1: Loop through each address exactly as they were originally created
.then(async geoPromises => {
let compactArray = []
for (const item of geoPromises) {
// Get the geo coded data
let city = await item;
// If data is available, store it
//console.log(city)
if(JSON.stringify(city) !== '{}') compactArray.push(city);
}
console.log(`Option 1 Results: ${compactArray.length}`)
//console.log(`Option 1 Results: `, compactArray)
// Proceed to Option 2
return geoPromises;
})
// Option 2: Promise All
.then(async geoPromises => {
let option2 = Promise.all(geoPromises)
.then(async array => {
// New array with Empty Objects removed
let compactArray = await array.filter(item => {
// If item is NOT an empty object, use it
return (JSON.stringify(item) !== '{}') ? true : false;
});
return compactArray;
})
.catch(err => err);
console.log(`Option 2: Results`, (await option2).length)
// Proceed to Option 3
return geoPromises
})
// Option 3: Async filter with Map. This provides a lot of control
// https://advancedweb.hu/how-to-use-async-functions-with-array-filter-in-javascript/
.then(async geoPromises => {
let mapAsync = async (arr, predicate) => {
return Promise.all(arr.map(predicate));
}
// Filter out the items in the array using the predicate
let filterAsync = async (array, predicate) => {
return mapAsync(array, predicate).then(filterMap => {
let newArr = array.filter((value, index) => filterMap[index]);
return newArr;
})
}
// The condition (predicate) that will test each item within the array
// Inspired by: https://stackoverflow.com/a/53508547
let isNotEmpty = async (obj) => {
var obj = await obj;
return !(Object.keys(obj).length === 0 && obj.constructor === Object);
}
let compactArray = await filterAsync(geoPromises, isNotEmpty)
.then(result => result)
.catch(Error => Error);
console.log("Option 3: Results", compactArray.length)
})
.catch(SearchCityNameError => SearchCityNameError);
})();
