Add Bottleneck limiter
This commit is contained in:
parent
533d1cadcd
commit
eb53cd7047
23
README.md
23
README.md
|
@ -266,6 +266,29 @@ Fetches the list of tags matching `params.q`. Results include both partial and f
|
|||
- q: the string to match
|
||||
- limit: the maximum number of results to return
|
||||
|
||||
## Rate Limiting
|
||||
|
||||
The API functions can be called with rate limiting like this:
|
||||
|
||||
```
|
||||
bcfetch.limiter.getAlbumInfo(...);
|
||||
```
|
||||
|
||||
[**Example**](examples/limiter.js) ([output](examples/limiter_output.txt))
|
||||
|
||||
Rate limiting is useful when you need to make a large number of queries and don't want to run the risk of getting rejected by the server for making too many requests within a short time interval. If you get a '429 Too Many Requests' error, then you should consider using the rate limiter.
|
||||
|
||||
The library uses [Bottleneck](https://www.npmjs.com/package/bottleneck) for rate limiting. You can configure the rate limiter like this:
|
||||
|
||||
```
|
||||
bcfetch.limiter.updateSettings({
|
||||
maxConcurrent: 10, // default: 5
|
||||
minTime: 100 // default: 200
|
||||
});
|
||||
```
|
||||
|
||||
`updateSettings()` is just a passthrough function to Bottleneck. Check the [Bottleneck doc](https://www.npmjs.com/package/bottleneck#docs) for the list of options you can set.
|
||||
|
||||
## Caching
|
||||
|
||||
The library maintains an in-memory cache for two types of resources:
|
||||
|
|
91
examples/limiter.js
Normal file
91
examples/limiter.js
Normal file
|
@ -0,0 +1,91 @@
|
|||
const bcfetch = require('../');
|
||||
|
||||
bcfetch.limiter.updateSettings({
|
||||
maxConcurrent: 10,
|
||||
minTime: 100
|
||||
});
|
||||
|
||||
const albumUrls = [
|
||||
'https://phoebebridgers.bandcamp.com/album/punisher',
|
||||
'https://mrsgreenbird.bandcamp.com/album/10-years-live',
|
||||
'https://ryleywalker.bandcamp.com/album/course-in-fable',
|
||||
'https://phoebebridgers.bandcamp.com/album/stranger-in-the-alps',
|
||||
'https://wearetyphoon.bandcamp.com/album/sympathetic-magic',
|
||||
'https://mariabc.bandcamp.com/album/devils-rain-2',
|
||||
'https://jimpullman.bandcamp.com/album/go-on-boldly',
|
||||
'https://debutants.bandcamp.com/album/indiana-newgrass-ep',
|
||||
'https://cassandrajenkins.bandcamp.com/album/an-overview-on-phenomenal-nature',
|
||||
'https://stefonaclearday.bandcamp.com/album/songs-of-love-and-unlove',
|
||||
'https://haleyheynderickx.bandcamp.com/album/i-need-to-start-a-garden',
|
||||
'https://phoebebridgers.bandcamp.com/album/copycat-killer',
|
||||
'https://grantleephillips.bandcamp.com/album/rag-town-pink-rebel',
|
||||
'https://garlandofhours.bandcamp.com/album/lucidia',
|
||||
'https://landehekt.bandcamp.com/album/going-to-hell',
|
||||
'https://emmaswift.bandcamp.com/album/blonde-on-the-tracks',
|
||||
'https://joannanewsom.bandcamp.com/album/ys',
|
||||
'https://joannanewsom.bandcamp.com/album/have-one-on-me',
|
||||
'https://saintseneca.bandcamp.com/album/all-youve-got-is-everyone',
|
||||
'https://phoebebridgers.bandcamp.com/album/if-we-make-it-through-december',
|
||||
'https://music.theohhellos.com/album/boreas',
|
||||
'https://rikkiwill.bandcamp.com/album/songs-for-rivers',
|
||||
'https://alexsalcido.bandcamp.com/album/im-a-bird',
|
||||
'https://helenadeland.bandcamp.com/album/someone-new',
|
||||
'https://ryleywalker.bandcamp.com/album/for-michael-ripps',
|
||||
'https://thebargain.bandcamp.com/album/yes-b-w-stay-awhile',
|
||||
'https://curtismcmurtry.bandcamp.com/album/toothless-messiah',
|
||||
'https://ceciliablairwright.bandcamp.com/album/another-human',
|
||||
'https://darrenhayman.bandcamp.com/album/music-to-watch-news-by',
|
||||
'https://virginiaastley.bandcamp.com/album/from-gardens-where-we-feel-secure',
|
||||
'https://radicalface.bandcamp.com/album/hidden-hollow-vol-one-singles',
|
||||
'https://gabriellepietrangelo.bandcamp.com/album/big-desert-sky',
|
||||
'https://miloandlovina.bandcamp.com/album/paper-hearts',
|
||||
'https://fatherjohnmisty.bandcamp.com/album/anthem-3-2',
|
||||
'https://thefauxpaws.bandcamp.com/album/the-hurricane-ep',
|
||||
'https://mrsgreenbird.bandcamp.com/album/dark-waters',
|
||||
'https://indigosparke.bandcamp.com/album/echo',
|
||||
'https://joannanewsom.bandcamp.com/album/divers',
|
||||
'https://kellymcfarling.bandcamp.com/album/deep-the-habit',
|
||||
'https://dogwood.bandcamp.com/album/the-imperfect-ep',
|
||||
'https://bedouine.bandcamp.com/album/bedouine',
|
||||
'https://craigmckerron.bandcamp.com/album/cabin-fever',
|
||||
'https://imskullcrusher.bandcamp.com/album/skullcrusher',
|
||||
'https://fleetfoxes.bandcamp.com/album/crack-up',
|
||||
'https://anothermichael.bandcamp.com/album/new-music-and-big-pop',
|
||||
'https://autourdelucie.bandcamp.com/album/bunker',
|
||||
'https://laurastevenson.bandcamp.com/album/sit-resist-remastered-deluxe-edition'
|
||||
];
|
||||
|
||||
const options = {
|
||||
albumImageFormat: 'art_app_large',
|
||||
artistImageFormat: 'bio_featured',
|
||||
includeRawData: false
|
||||
}
|
||||
|
||||
let fetchPromises = [];
|
||||
albumUrls.forEach( url => {
|
||||
fetchPromises.push(bcfetch.limiter.getAlbumInfo(url, options).then( result => {
|
||||
console.log(`Resolved: ${url}`)
|
||||
return result;
|
||||
}));
|
||||
});
|
||||
|
||||
console.log('Resolving album URLs with limiter...');
|
||||
Promise.all(fetchPromises).then( results => {
|
||||
console.log(`Total ${results.length} URLs resolved!`);
|
||||
console.log('');
|
||||
console.log('Now let\'s see what happens when we don\'t use limiter...');
|
||||
|
||||
fetchPromises = [];
|
||||
albumUrls.forEach( url => {
|
||||
fetchPromises.push(bcfetch.getAlbumInfo(url, options).then( result => {
|
||||
console.log(`Resolved: ${url}`)
|
||||
return result;
|
||||
}));
|
||||
});
|
||||
bcfetch.cache.clear();
|
||||
Promise.all(fetchPromises).then( results => {
|
||||
console.log(`Total ${results.length} URLs resolved!`);
|
||||
}).catch( error => {
|
||||
console.log(`An error occurred: ${error.message}`);
|
||||
});
|
||||
});
|
59
examples/limiter_output.txt
Normal file
59
examples/limiter_output.txt
Normal file
|
@ -0,0 +1,59 @@
|
|||
Resolving album URLs with limiter...
|
||||
Resolved: https://mrsgreenbird.bandcamp.com/album/10-years-live
|
||||
Resolved: https://phoebebridgers.bandcamp.com/album/punisher
|
||||
Resolved: https://phoebebridgers.bandcamp.com/album/stranger-in-the-alps
|
||||
Resolved: https://mariabc.bandcamp.com/album/devils-rain-2
|
||||
Resolved: https://ryleywalker.bandcamp.com/album/course-in-fable
|
||||
Resolved: https://wearetyphoon.bandcamp.com/album/sympathetic-magic
|
||||
Resolved: https://jimpullman.bandcamp.com/album/go-on-boldly
|
||||
Resolved: https://debutants.bandcamp.com/album/indiana-newgrass-ep
|
||||
Resolved: https://stefonaclearday.bandcamp.com/album/songs-of-love-and-unlove
|
||||
Resolved: https://cassandrajenkins.bandcamp.com/album/an-overview-on-phenomenal-nature
|
||||
Resolved: https://garlandofhours.bandcamp.com/album/lucidia
|
||||
Resolved: https://grantleephillips.bandcamp.com/album/rag-town-pink-rebel
|
||||
Resolved: https://phoebebridgers.bandcamp.com/album/copycat-killer
|
||||
Resolved: https://haleyheynderickx.bandcamp.com/album/i-need-to-start-a-garden
|
||||
Resolved: https://landehekt.bandcamp.com/album/going-to-hell
|
||||
Resolved: https://emmaswift.bandcamp.com/album/blonde-on-the-tracks
|
||||
Resolved: https://joannanewsom.bandcamp.com/album/ys
|
||||
Resolved: https://saintseneca.bandcamp.com/album/all-youve-got-is-everyone
|
||||
Resolved: https://phoebebridgers.bandcamp.com/album/if-we-make-it-through-december
|
||||
Resolved: https://music.theohhellos.com/album/boreas
|
||||
Resolved: https://rikkiwill.bandcamp.com/album/songs-for-rivers
|
||||
Resolved: https://alexsalcido.bandcamp.com/album/im-a-bird
|
||||
Resolved: https://joannanewsom.bandcamp.com/album/have-one-on-me
|
||||
Resolved: https://helenadeland.bandcamp.com/album/someone-new
|
||||
Resolved: https://thebargain.bandcamp.com/album/yes-b-w-stay-awhile
|
||||
Resolved: https://ryleywalker.bandcamp.com/album/for-michael-ripps
|
||||
Resolved: https://curtismcmurtry.bandcamp.com/album/toothless-messiah
|
||||
Resolved: https://ceciliablairwright.bandcamp.com/album/another-human
|
||||
Resolved: https://darrenhayman.bandcamp.com/album/music-to-watch-news-by
|
||||
Resolved: https://virginiaastley.bandcamp.com/album/from-gardens-where-we-feel-secure
|
||||
Resolved: https://radicalface.bandcamp.com/album/hidden-hollow-vol-one-singles
|
||||
Resolved: https://miloandlovina.bandcamp.com/album/paper-hearts
|
||||
Resolved: https://mrsgreenbird.bandcamp.com/album/dark-waters
|
||||
Resolved: https://gabriellepietrangelo.bandcamp.com/album/big-desert-sky
|
||||
Resolved: https://thefauxpaws.bandcamp.com/album/the-hurricane-ep
|
||||
Resolved: https://fatherjohnmisty.bandcamp.com/album/anthem-3-2
|
||||
Resolved: https://kellymcfarling.bandcamp.com/album/deep-the-habit
|
||||
Resolved: https://joannanewsom.bandcamp.com/album/divers
|
||||
Resolved: https://dogwood.bandcamp.com/album/the-imperfect-ep
|
||||
Resolved: https://indigosparke.bandcamp.com/album/echo
|
||||
Resolved: https://bedouine.bandcamp.com/album/bedouine
|
||||
Resolved: https://imskullcrusher.bandcamp.com/album/skullcrusher
|
||||
Resolved: https://fleetfoxes.bandcamp.com/album/crack-up
|
||||
Resolved: https://craigmckerron.bandcamp.com/album/cabin-fever
|
||||
Resolved: https://anothermichael.bandcamp.com/album/new-music-and-big-pop
|
||||
Resolved: https://autourdelucie.bandcamp.com/album/bunker
|
||||
Resolved: https://laurastevenson.bandcamp.com/album/sit-resist-remastered-deluxe-edition
|
||||
Total 47 URLs resolved!
|
||||
|
||||
Now let's see what happens when we don't use limiter...
|
||||
An error occurred: 429 Too Many Requests
|
||||
Resolved: https://cassandrajenkins.bandcamp.com/album/an-overview-on-phenomenal-nature
|
||||
Resolved: https://mrsgreenbird.bandcamp.com/album/10-years-live
|
||||
Resolved: https://thefauxpaws.bandcamp.com/album/the-hurricane-ep
|
||||
Resolved: https://mariabc.bandcamp.com/album/devils-rain-2
|
||||
Resolved: https://fleetfoxes.bandcamp.com/album/crack-up
|
||||
Resolved: https://ryleywalker.bandcamp.com/album/course-in-fable
|
||||
Resolved: https://curtismcmurtry.bandcamp.com/album/toothless-messiah
|
23
lib/index.js
23
lib/index.js
|
@ -1,4 +1,5 @@
|
|||
const fetch = require('node-fetch');
|
||||
const Bottleneck = require('bottleneck');
|
||||
const utils = require('./utils.js');
|
||||
const parser = require('./parser.js');
|
||||
const Cache = require('./cache.js');
|
||||
|
@ -414,7 +415,7 @@ async function _fetchPage(url, json = false, fetchOptions = null) {
|
|||
else {
|
||||
return json ? res.json() : res.text();
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
|
@ -435,7 +436,8 @@ const cache = {
|
|||
clear: _cache.clear.bind(_cache)
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
// Exported functions
|
||||
const _exportFn = {
|
||||
discover,
|
||||
getDiscoverOptions,
|
||||
sanitizeDiscoverParams,
|
||||
|
@ -449,7 +451,6 @@ module.exports = {
|
|||
search,
|
||||
getAlbumHighlightsByTag,
|
||||
getTags,
|
||||
cache,
|
||||
getAllShows,
|
||||
getShow,
|
||||
getArticleCategories,
|
||||
|
@ -460,4 +461,18 @@ module.exports = {
|
|||
getReleasesByTag,
|
||||
searchTag,
|
||||
searchLocation
|
||||
};
|
||||
};
|
||||
|
||||
// Bottleneck limiter
|
||||
const _limiter = new Bottleneck({
|
||||
maxConcurrent: 5,
|
||||
minTime: 200
|
||||
});
|
||||
const limiter = {};
|
||||
for (const [fnName, fn] of Object.entries(_exportFn)) {
|
||||
limiter[fnName] = _limiter.wrap(fn);
|
||||
}
|
||||
limiter.updateSettings = _limiter.updateSettings.bind(_limiter);
|
||||
|
||||
// Module exports
|
||||
module.exports = Object.assign({}, _exportFn, { cache, limiter });
|
||||
|
|
|
@ -22,6 +22,7 @@
|
|||
"author": "Patrick Kan",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"bottleneck": "^2.19.5",
|
||||
"cheerio": "^1.0.0-rc.5",
|
||||
"html-entities": "^2.0.2",
|
||||
"node-cache": "^5.1.2",
|
||||
|
|
Loading…
Reference in New Issue
Block a user