From eb53cd70475ca50c5211a0afd97ff5e414ffacfc Mon Sep 17 00:00:00 2001 From: patrickkfkan Date: Tue, 16 Feb 2021 20:33:54 +0800 Subject: [PATCH] Add Bottleneck limiter --- README.md | 23 ++++++++++ examples/limiter.js | 91 +++++++++++++++++++++++++++++++++++++ examples/limiter_output.txt | 59 ++++++++++++++++++++++++ lib/index.js | 23 ++++++++-- package.json | 1 + 5 files changed, 193 insertions(+), 4 deletions(-) create mode 100644 examples/limiter.js create mode 100644 examples/limiter_output.txt diff --git a/README.md b/README.md index 5431140..cdbf82e 100644 --- a/README.md +++ b/README.md @@ -266,6 +266,29 @@ Fetches the list of tags matching `params.q`. Results include both partial and f - q: the string to match - limit: the maximum number of results to return +## Rate Limiting + +The API functions can be called with rate limiting like this: + +``` +bcfetch.limiter.getAlbumInfo(...); +``` + +[**Example**](examples/limiter.js) ([output](examples/limiter_output.txt)) + +Rate limiting is useful when you need to make a large number of queries and don't want to run the risk of getting rejected by the server for making too many requests within a short time interval. If you get a '429 Too Many Requests' error, then you should consider using the rate limiter. + +The library uses [Bottleneck](https://www.npmjs.com/package/bottleneck) for rate limiting. You can configure the rate limiter like this: + +``` +bcfetch.limiter.updateSettings({ + maxConcurrent: 10, // default: 5 + minTime: 100 // default: 200 +}); +``` + +`updateSettings()` is just a passthrough function to Bottleneck. Check the [Bottleneck doc](https://www.npmjs.com/package/bottleneck#docs) for the list of options you can set. + ## Caching The library maintains an in-memory cache for two types of resources: diff --git a/examples/limiter.js b/examples/limiter.js new file mode 100644 index 0000000..33beb56 --- /dev/null +++ b/examples/limiter.js @@ -0,0 +1,91 @@ +const bcfetch = require('../'); + +bcfetch.limiter.updateSettings({ + maxConcurrent: 10, + minTime: 100 +}); + +const albumUrls = [ + 'https://phoebebridgers.bandcamp.com/album/punisher', + 'https://mrsgreenbird.bandcamp.com/album/10-years-live', + 'https://ryleywalker.bandcamp.com/album/course-in-fable', + 'https://phoebebridgers.bandcamp.com/album/stranger-in-the-alps', + 'https://wearetyphoon.bandcamp.com/album/sympathetic-magic', + 'https://mariabc.bandcamp.com/album/devils-rain-2', + 'https://jimpullman.bandcamp.com/album/go-on-boldly', + 'https://debutants.bandcamp.com/album/indiana-newgrass-ep', + 'https://cassandrajenkins.bandcamp.com/album/an-overview-on-phenomenal-nature', + 'https://stefonaclearday.bandcamp.com/album/songs-of-love-and-unlove', + 'https://haleyheynderickx.bandcamp.com/album/i-need-to-start-a-garden', + 'https://phoebebridgers.bandcamp.com/album/copycat-killer', + 'https://grantleephillips.bandcamp.com/album/rag-town-pink-rebel', + 'https://garlandofhours.bandcamp.com/album/lucidia', + 'https://landehekt.bandcamp.com/album/going-to-hell', + 'https://emmaswift.bandcamp.com/album/blonde-on-the-tracks', + 'https://joannanewsom.bandcamp.com/album/ys', + 'https://joannanewsom.bandcamp.com/album/have-one-on-me', + 'https://saintseneca.bandcamp.com/album/all-youve-got-is-everyone', + 'https://phoebebridgers.bandcamp.com/album/if-we-make-it-through-december', + 'https://music.theohhellos.com/album/boreas', + 'https://rikkiwill.bandcamp.com/album/songs-for-rivers', + 'https://alexsalcido.bandcamp.com/album/im-a-bird', + 'https://helenadeland.bandcamp.com/album/someone-new', + 'https://ryleywalker.bandcamp.com/album/for-michael-ripps', + 'https://thebargain.bandcamp.com/album/yes-b-w-stay-awhile', + 'https://curtismcmurtry.bandcamp.com/album/toothless-messiah', + 'https://ceciliablairwright.bandcamp.com/album/another-human', + 'https://darrenhayman.bandcamp.com/album/music-to-watch-news-by', + 'https://virginiaastley.bandcamp.com/album/from-gardens-where-we-feel-secure', + 'https://radicalface.bandcamp.com/album/hidden-hollow-vol-one-singles', + 'https://gabriellepietrangelo.bandcamp.com/album/big-desert-sky', + 'https://miloandlovina.bandcamp.com/album/paper-hearts', + 'https://fatherjohnmisty.bandcamp.com/album/anthem-3-2', + 'https://thefauxpaws.bandcamp.com/album/the-hurricane-ep', + 'https://mrsgreenbird.bandcamp.com/album/dark-waters', + 'https://indigosparke.bandcamp.com/album/echo', + 'https://joannanewsom.bandcamp.com/album/divers', + 'https://kellymcfarling.bandcamp.com/album/deep-the-habit', + 'https://dogwood.bandcamp.com/album/the-imperfect-ep', + 'https://bedouine.bandcamp.com/album/bedouine', + 'https://craigmckerron.bandcamp.com/album/cabin-fever', + 'https://imskullcrusher.bandcamp.com/album/skullcrusher', + 'https://fleetfoxes.bandcamp.com/album/crack-up', + 'https://anothermichael.bandcamp.com/album/new-music-and-big-pop', + 'https://autourdelucie.bandcamp.com/album/bunker', + 'https://laurastevenson.bandcamp.com/album/sit-resist-remastered-deluxe-edition' +]; + +const options = { + albumImageFormat: 'art_app_large', + artistImageFormat: 'bio_featured', + includeRawData: false +} + +let fetchPromises = []; +albumUrls.forEach( url => { + fetchPromises.push(bcfetch.limiter.getAlbumInfo(url, options).then( result => { + console.log(`Resolved: ${url}`) + return result; + })); +}); + +console.log('Resolving album URLs with limiter...'); +Promise.all(fetchPromises).then( results => { + console.log(`Total ${results.length} URLs resolved!`); + console.log(''); + console.log('Now let\'s see what happens when we don\'t use limiter...'); + + fetchPromises = []; + albumUrls.forEach( url => { + fetchPromises.push(bcfetch.getAlbumInfo(url, options).then( result => { + console.log(`Resolved: ${url}`) + return result; + })); + }); + bcfetch.cache.clear(); + Promise.all(fetchPromises).then( results => { + console.log(`Total ${results.length} URLs resolved!`); + }).catch( error => { + console.log(`An error occurred: ${error.message}`); + }); +}); \ No newline at end of file diff --git a/examples/limiter_output.txt b/examples/limiter_output.txt new file mode 100644 index 0000000..d2de90e --- /dev/null +++ b/examples/limiter_output.txt @@ -0,0 +1,59 @@ +Resolving album URLs with limiter... +Resolved: https://mrsgreenbird.bandcamp.com/album/10-years-live +Resolved: https://phoebebridgers.bandcamp.com/album/punisher +Resolved: https://phoebebridgers.bandcamp.com/album/stranger-in-the-alps +Resolved: https://mariabc.bandcamp.com/album/devils-rain-2 +Resolved: https://ryleywalker.bandcamp.com/album/course-in-fable +Resolved: https://wearetyphoon.bandcamp.com/album/sympathetic-magic +Resolved: https://jimpullman.bandcamp.com/album/go-on-boldly +Resolved: https://debutants.bandcamp.com/album/indiana-newgrass-ep +Resolved: https://stefonaclearday.bandcamp.com/album/songs-of-love-and-unlove +Resolved: https://cassandrajenkins.bandcamp.com/album/an-overview-on-phenomenal-nature +Resolved: https://garlandofhours.bandcamp.com/album/lucidia +Resolved: https://grantleephillips.bandcamp.com/album/rag-town-pink-rebel +Resolved: https://phoebebridgers.bandcamp.com/album/copycat-killer +Resolved: https://haleyheynderickx.bandcamp.com/album/i-need-to-start-a-garden +Resolved: https://landehekt.bandcamp.com/album/going-to-hell +Resolved: https://emmaswift.bandcamp.com/album/blonde-on-the-tracks +Resolved: https://joannanewsom.bandcamp.com/album/ys +Resolved: https://saintseneca.bandcamp.com/album/all-youve-got-is-everyone +Resolved: https://phoebebridgers.bandcamp.com/album/if-we-make-it-through-december +Resolved: https://music.theohhellos.com/album/boreas +Resolved: https://rikkiwill.bandcamp.com/album/songs-for-rivers +Resolved: https://alexsalcido.bandcamp.com/album/im-a-bird +Resolved: https://joannanewsom.bandcamp.com/album/have-one-on-me +Resolved: https://helenadeland.bandcamp.com/album/someone-new +Resolved: https://thebargain.bandcamp.com/album/yes-b-w-stay-awhile +Resolved: https://ryleywalker.bandcamp.com/album/for-michael-ripps +Resolved: https://curtismcmurtry.bandcamp.com/album/toothless-messiah +Resolved: https://ceciliablairwright.bandcamp.com/album/another-human +Resolved: https://darrenhayman.bandcamp.com/album/music-to-watch-news-by +Resolved: https://virginiaastley.bandcamp.com/album/from-gardens-where-we-feel-secure +Resolved: https://radicalface.bandcamp.com/album/hidden-hollow-vol-one-singles +Resolved: https://miloandlovina.bandcamp.com/album/paper-hearts +Resolved: https://mrsgreenbird.bandcamp.com/album/dark-waters +Resolved: https://gabriellepietrangelo.bandcamp.com/album/big-desert-sky +Resolved: https://thefauxpaws.bandcamp.com/album/the-hurricane-ep +Resolved: https://fatherjohnmisty.bandcamp.com/album/anthem-3-2 +Resolved: https://kellymcfarling.bandcamp.com/album/deep-the-habit +Resolved: https://joannanewsom.bandcamp.com/album/divers +Resolved: https://dogwood.bandcamp.com/album/the-imperfect-ep +Resolved: https://indigosparke.bandcamp.com/album/echo +Resolved: https://bedouine.bandcamp.com/album/bedouine +Resolved: https://imskullcrusher.bandcamp.com/album/skullcrusher +Resolved: https://fleetfoxes.bandcamp.com/album/crack-up +Resolved: https://craigmckerron.bandcamp.com/album/cabin-fever +Resolved: https://anothermichael.bandcamp.com/album/new-music-and-big-pop +Resolved: https://autourdelucie.bandcamp.com/album/bunker +Resolved: https://laurastevenson.bandcamp.com/album/sit-resist-remastered-deluxe-edition +Total 47 URLs resolved! + +Now let's see what happens when we don't use limiter... +An error occurred: 429 Too Many Requests +Resolved: https://cassandrajenkins.bandcamp.com/album/an-overview-on-phenomenal-nature +Resolved: https://mrsgreenbird.bandcamp.com/album/10-years-live +Resolved: https://thefauxpaws.bandcamp.com/album/the-hurricane-ep +Resolved: https://mariabc.bandcamp.com/album/devils-rain-2 +Resolved: https://fleetfoxes.bandcamp.com/album/crack-up +Resolved: https://ryleywalker.bandcamp.com/album/course-in-fable +Resolved: https://curtismcmurtry.bandcamp.com/album/toothless-messiah diff --git a/lib/index.js b/lib/index.js index 56667c0..1e515a1 100644 --- a/lib/index.js +++ b/lib/index.js @@ -1,4 +1,5 @@ const fetch = require('node-fetch'); +const Bottleneck = require('bottleneck'); const utils = require('./utils.js'); const parser = require('./parser.js'); const Cache = require('./cache.js'); @@ -414,7 +415,7 @@ async function _fetchPage(url, json = false, fetchOptions = null) { else { return json ? res.json() : res.text(); } - }); + }); }); } @@ -435,7 +436,8 @@ const cache = { clear: _cache.clear.bind(_cache) }; -module.exports = { +// Exported functions +const _exportFn = { discover, getDiscoverOptions, sanitizeDiscoverParams, @@ -449,7 +451,6 @@ module.exports = { search, getAlbumHighlightsByTag, getTags, - cache, getAllShows, getShow, getArticleCategories, @@ -460,4 +461,18 @@ module.exports = { getReleasesByTag, searchTag, searchLocation -}; \ No newline at end of file +}; + +// Bottleneck limiter +const _limiter = new Bottleneck({ + maxConcurrent: 5, + minTime: 200 +}); +const limiter = {}; +for (const [fnName, fn] of Object.entries(_exportFn)) { + limiter[fnName] = _limiter.wrap(fn); +} +limiter.updateSettings = _limiter.updateSettings.bind(_limiter); + +// Module exports +module.exports = Object.assign({}, _exportFn, { cache, limiter }); diff --git a/package.json b/package.json index 19136eb..3f04acc 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "author": "Patrick Kan", "license": "MIT", "dependencies": { + "bottleneck": "^2.19.5", "cheerio": "^1.0.0-rc.5", "html-entities": "^2.0.2", "node-cache": "^5.1.2",