Add Bottleneck limiter

README.md

@@ -266,6 +266,29 @@ Fetches the list of tags matching `params.q`. Results include both partial and f
     - q: the string to match
     - limit: the maximum number of results to return
 
+## Rate Limiting
+
+The API functions can be called with rate limiting like this:
+
+```
+bcfetch.limiter.getAlbumInfo(...);
+```
+
+[**Example**](examples/limiter.js) ([output](examples/limiter_output.txt))
+
+Rate limiting is useful when you need to make a large number of queries and don't want to run the risk of getting rejected by the server for making too many requests within a short time interval. If you get a '429 Too Many Requests' error, then you should consider using the rate limiter.
+
+The library uses [Bottleneck](https://www.npmjs.com/package/bottleneck) for rate limiting. You can configure the rate limiter like this:
+
+```
+bcfetch.limiter.updateSettings({
+    maxConcurrent: 10,  // default: 5
+    minTime: 100        // default: 200
+});
+```
+
+`updateSettings()` is just a passthrough function to Bottleneck. Check the [Bottleneck doc](https://www.npmjs.com/package/bottleneck#docs) for the list of options you can set.
+
 ## Caching
 
 The library maintains an in-memory cache for two types of resources:

examples/limiter.js

@@ -0,0 +1,91 @@
+const bcfetch = require('../');
+
+bcfetch.limiter.updateSettings({
+    maxConcurrent: 10,
+    minTime: 100
+});
+
+const albumUrls = [
+    'https://phoebebridgers.bandcamp.com/album/punisher',
+    'https://mrsgreenbird.bandcamp.com/album/10-years-live',
+    'https://ryleywalker.bandcamp.com/album/course-in-fable',
+    'https://phoebebridgers.bandcamp.com/album/stranger-in-the-alps',
+    'https://wearetyphoon.bandcamp.com/album/sympathetic-magic',
+    'https://mariabc.bandcamp.com/album/devils-rain-2',
+    'https://jimpullman.bandcamp.com/album/go-on-boldly',
+    'https://debutants.bandcamp.com/album/indiana-newgrass-ep',
+    'https://cassandrajenkins.bandcamp.com/album/an-overview-on-phenomenal-nature',
+    'https://stefonaclearday.bandcamp.com/album/songs-of-love-and-unlove',
+    'https://haleyheynderickx.bandcamp.com/album/i-need-to-start-a-garden',
+    'https://phoebebridgers.bandcamp.com/album/copycat-killer',
+    'https://grantleephillips.bandcamp.com/album/rag-town-pink-rebel',
+    'https://garlandofhours.bandcamp.com/album/lucidia',
+    'https://landehekt.bandcamp.com/album/going-to-hell',
+    'https://emmaswift.bandcamp.com/album/blonde-on-the-tracks',
+    'https://joannanewsom.bandcamp.com/album/ys',
+    'https://joannanewsom.bandcamp.com/album/have-one-on-me',
+    'https://saintseneca.bandcamp.com/album/all-youve-got-is-everyone',
+    'https://phoebebridgers.bandcamp.com/album/if-we-make-it-through-december',
+    'https://music.theohhellos.com/album/boreas',
+    'https://rikkiwill.bandcamp.com/album/songs-for-rivers',
+    'https://alexsalcido.bandcamp.com/album/im-a-bird',
+    'https://helenadeland.bandcamp.com/album/someone-new',
+    'https://ryleywalker.bandcamp.com/album/for-michael-ripps',
+    'https://thebargain.bandcamp.com/album/yes-b-w-stay-awhile',
+    'https://curtismcmurtry.bandcamp.com/album/toothless-messiah',
+    'https://ceciliablairwright.bandcamp.com/album/another-human',
+    'https://darrenhayman.bandcamp.com/album/music-to-watch-news-by',
+    'https://virginiaastley.bandcamp.com/album/from-gardens-where-we-feel-secure',
+    'https://radicalface.bandcamp.com/album/hidden-hollow-vol-one-singles',
+    'https://gabriellepietrangelo.bandcamp.com/album/big-desert-sky',
+    'https://miloandlovina.bandcamp.com/album/paper-hearts',
+    'https://fatherjohnmisty.bandcamp.com/album/anthem-3-2',
+    'https://thefauxpaws.bandcamp.com/album/the-hurricane-ep',
+    'https://mrsgreenbird.bandcamp.com/album/dark-waters',
+    'https://indigosparke.bandcamp.com/album/echo',
+    'https://joannanewsom.bandcamp.com/album/divers',
+    'https://kellymcfarling.bandcamp.com/album/deep-the-habit',
+    'https://dogwood.bandcamp.com/album/the-imperfect-ep',
+    'https://bedouine.bandcamp.com/album/bedouine',
+    'https://craigmckerron.bandcamp.com/album/cabin-fever',
+    'https://imskullcrusher.bandcamp.com/album/skullcrusher',
+    'https://fleetfoxes.bandcamp.com/album/crack-up',
+    'https://anothermichael.bandcamp.com/album/new-music-and-big-pop',
+    'https://autourdelucie.bandcamp.com/album/bunker',
+    'https://laurastevenson.bandcamp.com/album/sit-resist-remastered-deluxe-edition'
+];
+
+const options = {
+    albumImageFormat: 'art_app_large',
+    artistImageFormat: 'bio_featured',
+    includeRawData: false
+}
+
+let fetchPromises = [];
+albumUrls.forEach( url => {
+    fetchPromises.push(bcfetch.limiter.getAlbumInfo(url, options).then( result => {
+        console.log(`Resolved: ${url}`)
+        return result;
+    }));
+});
+
+console.log('Resolving album URLs with limiter...');
+Promise.all(fetchPromises).then( results => {
+    console.log(`Total ${results.length} URLs resolved!`);
+    console.log('');
+    console.log('Now let\'s see what happens when we don\'t use limiter...');
+
+    fetchPromises = [];
+    albumUrls.forEach( url => {
+        fetchPromises.push(bcfetch.getAlbumInfo(url, options).then( result => {
+            console.log(`Resolved: ${url}`)
+            return result;
+        }));
+    });
+    bcfetch.cache.clear();
+    Promise.all(fetchPromises).then( results => {
+        console.log(`Total ${results.length} URLs resolved!`);
+    }).catch( error => {
+        console.log(`An error occurred: ${error.message}`);
+    });
+});

examples/limiter_output.txt

@@ -0,0 +1,59 @@
+Resolving album URLs with limiter...
+Resolved: https://mrsgreenbird.bandcamp.com/album/10-years-live
+Resolved: https://phoebebridgers.bandcamp.com/album/punisher
+Resolved: https://phoebebridgers.bandcamp.com/album/stranger-in-the-alps
+Resolved: https://mariabc.bandcamp.com/album/devils-rain-2
+Resolved: https://ryleywalker.bandcamp.com/album/course-in-fable
+Resolved: https://wearetyphoon.bandcamp.com/album/sympathetic-magic
+Resolved: https://jimpullman.bandcamp.com/album/go-on-boldly
+Resolved: https://debutants.bandcamp.com/album/indiana-newgrass-ep
+Resolved: https://stefonaclearday.bandcamp.com/album/songs-of-love-and-unlove
+Resolved: https://cassandrajenkins.bandcamp.com/album/an-overview-on-phenomenal-nature
+Resolved: https://garlandofhours.bandcamp.com/album/lucidia
+Resolved: https://grantleephillips.bandcamp.com/album/rag-town-pink-rebel
+Resolved: https://phoebebridgers.bandcamp.com/album/copycat-killer
+Resolved: https://haleyheynderickx.bandcamp.com/album/i-need-to-start-a-garden
+Resolved: https://landehekt.bandcamp.com/album/going-to-hell
+Resolved: https://emmaswift.bandcamp.com/album/blonde-on-the-tracks
+Resolved: https://joannanewsom.bandcamp.com/album/ys
+Resolved: https://saintseneca.bandcamp.com/album/all-youve-got-is-everyone
+Resolved: https://phoebebridgers.bandcamp.com/album/if-we-make-it-through-december
+Resolved: https://music.theohhellos.com/album/boreas
+Resolved: https://rikkiwill.bandcamp.com/album/songs-for-rivers
+Resolved: https://alexsalcido.bandcamp.com/album/im-a-bird
+Resolved: https://joannanewsom.bandcamp.com/album/have-one-on-me
+Resolved: https://helenadeland.bandcamp.com/album/someone-new
+Resolved: https://thebargain.bandcamp.com/album/yes-b-w-stay-awhile
+Resolved: https://ryleywalker.bandcamp.com/album/for-michael-ripps
+Resolved: https://curtismcmurtry.bandcamp.com/album/toothless-messiah
+Resolved: https://ceciliablairwright.bandcamp.com/album/another-human
+Resolved: https://darrenhayman.bandcamp.com/album/music-to-watch-news-by
+Resolved: https://virginiaastley.bandcamp.com/album/from-gardens-where-we-feel-secure
+Resolved: https://radicalface.bandcamp.com/album/hidden-hollow-vol-one-singles
+Resolved: https://miloandlovina.bandcamp.com/album/paper-hearts
+Resolved: https://mrsgreenbird.bandcamp.com/album/dark-waters
+Resolved: https://gabriellepietrangelo.bandcamp.com/album/big-desert-sky
+Resolved: https://thefauxpaws.bandcamp.com/album/the-hurricane-ep
+Resolved: https://fatherjohnmisty.bandcamp.com/album/anthem-3-2
+Resolved: https://kellymcfarling.bandcamp.com/album/deep-the-habit
+Resolved: https://joannanewsom.bandcamp.com/album/divers
+Resolved: https://dogwood.bandcamp.com/album/the-imperfect-ep
+Resolved: https://indigosparke.bandcamp.com/album/echo
+Resolved: https://bedouine.bandcamp.com/album/bedouine
+Resolved: https://imskullcrusher.bandcamp.com/album/skullcrusher
+Resolved: https://fleetfoxes.bandcamp.com/album/crack-up
+Resolved: https://craigmckerron.bandcamp.com/album/cabin-fever
+Resolved: https://anothermichael.bandcamp.com/album/new-music-and-big-pop
+Resolved: https://autourdelucie.bandcamp.com/album/bunker
+Resolved: https://laurastevenson.bandcamp.com/album/sit-resist-remastered-deluxe-edition
+Total 47 URLs resolved!
+
+Now let's see what happens when we don't use limiter...
+An error occurred: 429 Too Many Requests
+Resolved: https://cassandrajenkins.bandcamp.com/album/an-overview-on-phenomenal-nature
+Resolved: https://mrsgreenbird.bandcamp.com/album/10-years-live
+Resolved: https://thefauxpaws.bandcamp.com/album/the-hurricane-ep
+Resolved: https://mariabc.bandcamp.com/album/devils-rain-2
+Resolved: https://fleetfoxes.bandcamp.com/album/crack-up
+Resolved: https://ryleywalker.bandcamp.com/album/course-in-fable
+Resolved: https://curtismcmurtry.bandcamp.com/album/toothless-messiah

lib/index.js

@@ -1,4 +1,5 @@
 const fetch = require('node-fetch');
+const Bottleneck = require('bottleneck');
 const utils = require('./utils.js');
 const parser = require('./parser.js');
 const Cache = require('./cache.js');
@@ -435,7 +436,8 @@ const cache = {
     clear: _cache.clear.bind(_cache)
 };
 
-module.exports = {
+// Exported functions
+const _exportFn = {
     discover,
     getDiscoverOptions,
     sanitizeDiscoverParams,
@@ -449,7 +451,6 @@ module.exports = {
     search,
     getAlbumHighlightsByTag,
     getTags,
-    cache,
     getAllShows,
     getShow,
     getArticleCategories,
@@ -461,3 +462,17 @@ module.exports = {
     searchTag,
     searchLocation
 };
+
+// Bottleneck limiter
+const _limiter = new Bottleneck({
+    maxConcurrent: 5,
+    minTime: 200
+});
+const limiter = {};
+for (const [fnName, fn] of Object.entries(_exportFn)) {
+    limiter[fnName] = _limiter.wrap(fn);
+}
+limiter.updateSettings = _limiter.updateSettings.bind(_limiter);
+
+// Module exports
+module.exports = Object.assign({}, _exportFn, { cache, limiter });

package.json

@@ -22,6 +22,7 @@
   "author": "Patrick Kan",
   "license": "MIT",
   "dependencies": {
+    "bottleneck": "^2.19.5",
     "cheerio": "^1.0.0-rc.5",
     "html-entities": "^2.0.2",
     "node-cache": "^5.1.2",