Add Bottleneck limiter

This commit is contained in:
patrickkfkan 2021-02-16 20:33:54 +08:00
parent 533d1cadcd
commit eb53cd7047
5 changed files with 193 additions and 4 deletions

View File

@ -266,6 +266,29 @@ Fetches the list of tags matching `params.q`. Results include both partial and f
- q: the string to match
- limit: the maximum number of results to return
## Rate Limiting
The API functions can be called with rate limiting like this:
```
bcfetch.limiter.getAlbumInfo(...);
```
[**Example**](examples/limiter.js) ([output](examples/limiter_output.txt))
Rate limiting is useful when you need to make a large number of queries and don't want to run the risk of getting rejected by the server for making too many requests within a short time interval. If you get a '429 Too Many Requests' error, then you should consider using the rate limiter.
The library uses [Bottleneck](https://www.npmjs.com/package/bottleneck) for rate limiting. You can configure the rate limiter like this:
```
bcfetch.limiter.updateSettings({
maxConcurrent: 10, // default: 5
minTime: 100 // default: 200
});
```
`updateSettings()` is just a passthrough function to Bottleneck. Check the [Bottleneck doc](https://www.npmjs.com/package/bottleneck#docs) for the list of options you can set.
## Caching
The library maintains an in-memory cache for two types of resources:

91
examples/limiter.js Normal file
View File

@ -0,0 +1,91 @@
const bcfetch = require('../');
bcfetch.limiter.updateSettings({
maxConcurrent: 10,
minTime: 100
});
const albumUrls = [
'https://phoebebridgers.bandcamp.com/album/punisher',
'https://mrsgreenbird.bandcamp.com/album/10-years-live',
'https://ryleywalker.bandcamp.com/album/course-in-fable',
'https://phoebebridgers.bandcamp.com/album/stranger-in-the-alps',
'https://wearetyphoon.bandcamp.com/album/sympathetic-magic',
'https://mariabc.bandcamp.com/album/devils-rain-2',
'https://jimpullman.bandcamp.com/album/go-on-boldly',
'https://debutants.bandcamp.com/album/indiana-newgrass-ep',
'https://cassandrajenkins.bandcamp.com/album/an-overview-on-phenomenal-nature',
'https://stefonaclearday.bandcamp.com/album/songs-of-love-and-unlove',
'https://haleyheynderickx.bandcamp.com/album/i-need-to-start-a-garden',
'https://phoebebridgers.bandcamp.com/album/copycat-killer',
'https://grantleephillips.bandcamp.com/album/rag-town-pink-rebel',
'https://garlandofhours.bandcamp.com/album/lucidia',
'https://landehekt.bandcamp.com/album/going-to-hell',
'https://emmaswift.bandcamp.com/album/blonde-on-the-tracks',
'https://joannanewsom.bandcamp.com/album/ys',
'https://joannanewsom.bandcamp.com/album/have-one-on-me',
'https://saintseneca.bandcamp.com/album/all-youve-got-is-everyone',
'https://phoebebridgers.bandcamp.com/album/if-we-make-it-through-december',
'https://music.theohhellos.com/album/boreas',
'https://rikkiwill.bandcamp.com/album/songs-for-rivers',
'https://alexsalcido.bandcamp.com/album/im-a-bird',
'https://helenadeland.bandcamp.com/album/someone-new',
'https://ryleywalker.bandcamp.com/album/for-michael-ripps',
'https://thebargain.bandcamp.com/album/yes-b-w-stay-awhile',
'https://curtismcmurtry.bandcamp.com/album/toothless-messiah',
'https://ceciliablairwright.bandcamp.com/album/another-human',
'https://darrenhayman.bandcamp.com/album/music-to-watch-news-by',
'https://virginiaastley.bandcamp.com/album/from-gardens-where-we-feel-secure',
'https://radicalface.bandcamp.com/album/hidden-hollow-vol-one-singles',
'https://gabriellepietrangelo.bandcamp.com/album/big-desert-sky',
'https://miloandlovina.bandcamp.com/album/paper-hearts',
'https://fatherjohnmisty.bandcamp.com/album/anthem-3-2',
'https://thefauxpaws.bandcamp.com/album/the-hurricane-ep',
'https://mrsgreenbird.bandcamp.com/album/dark-waters',
'https://indigosparke.bandcamp.com/album/echo',
'https://joannanewsom.bandcamp.com/album/divers',
'https://kellymcfarling.bandcamp.com/album/deep-the-habit',
'https://dogwood.bandcamp.com/album/the-imperfect-ep',
'https://bedouine.bandcamp.com/album/bedouine',
'https://craigmckerron.bandcamp.com/album/cabin-fever',
'https://imskullcrusher.bandcamp.com/album/skullcrusher',
'https://fleetfoxes.bandcamp.com/album/crack-up',
'https://anothermichael.bandcamp.com/album/new-music-and-big-pop',
'https://autourdelucie.bandcamp.com/album/bunker',
'https://laurastevenson.bandcamp.com/album/sit-resist-remastered-deluxe-edition'
];
const options = {
albumImageFormat: 'art_app_large',
artistImageFormat: 'bio_featured',
includeRawData: false
}
let fetchPromises = [];
albumUrls.forEach( url => {
fetchPromises.push(bcfetch.limiter.getAlbumInfo(url, options).then( result => {
console.log(`Resolved: ${url}`)
return result;
}));
});
console.log('Resolving album URLs with limiter...');
Promise.all(fetchPromises).then( results => {
console.log(`Total ${results.length} URLs resolved!`);
console.log('');
console.log('Now let\'s see what happens when we don\'t use limiter...');
fetchPromises = [];
albumUrls.forEach( url => {
fetchPromises.push(bcfetch.getAlbumInfo(url, options).then( result => {
console.log(`Resolved: ${url}`)
return result;
}));
});
bcfetch.cache.clear();
Promise.all(fetchPromises).then( results => {
console.log(`Total ${results.length} URLs resolved!`);
}).catch( error => {
console.log(`An error occurred: ${error.message}`);
});
});

View File

@ -0,0 +1,59 @@
Resolving album URLs with limiter...
Resolved: https://mrsgreenbird.bandcamp.com/album/10-years-live
Resolved: https://phoebebridgers.bandcamp.com/album/punisher
Resolved: https://phoebebridgers.bandcamp.com/album/stranger-in-the-alps
Resolved: https://mariabc.bandcamp.com/album/devils-rain-2
Resolved: https://ryleywalker.bandcamp.com/album/course-in-fable
Resolved: https://wearetyphoon.bandcamp.com/album/sympathetic-magic
Resolved: https://jimpullman.bandcamp.com/album/go-on-boldly
Resolved: https://debutants.bandcamp.com/album/indiana-newgrass-ep
Resolved: https://stefonaclearday.bandcamp.com/album/songs-of-love-and-unlove
Resolved: https://cassandrajenkins.bandcamp.com/album/an-overview-on-phenomenal-nature
Resolved: https://garlandofhours.bandcamp.com/album/lucidia
Resolved: https://grantleephillips.bandcamp.com/album/rag-town-pink-rebel
Resolved: https://phoebebridgers.bandcamp.com/album/copycat-killer
Resolved: https://haleyheynderickx.bandcamp.com/album/i-need-to-start-a-garden
Resolved: https://landehekt.bandcamp.com/album/going-to-hell
Resolved: https://emmaswift.bandcamp.com/album/blonde-on-the-tracks
Resolved: https://joannanewsom.bandcamp.com/album/ys
Resolved: https://saintseneca.bandcamp.com/album/all-youve-got-is-everyone
Resolved: https://phoebebridgers.bandcamp.com/album/if-we-make-it-through-december
Resolved: https://music.theohhellos.com/album/boreas
Resolved: https://rikkiwill.bandcamp.com/album/songs-for-rivers
Resolved: https://alexsalcido.bandcamp.com/album/im-a-bird
Resolved: https://joannanewsom.bandcamp.com/album/have-one-on-me
Resolved: https://helenadeland.bandcamp.com/album/someone-new
Resolved: https://thebargain.bandcamp.com/album/yes-b-w-stay-awhile
Resolved: https://ryleywalker.bandcamp.com/album/for-michael-ripps
Resolved: https://curtismcmurtry.bandcamp.com/album/toothless-messiah
Resolved: https://ceciliablairwright.bandcamp.com/album/another-human
Resolved: https://darrenhayman.bandcamp.com/album/music-to-watch-news-by
Resolved: https://virginiaastley.bandcamp.com/album/from-gardens-where-we-feel-secure
Resolved: https://radicalface.bandcamp.com/album/hidden-hollow-vol-one-singles
Resolved: https://miloandlovina.bandcamp.com/album/paper-hearts
Resolved: https://mrsgreenbird.bandcamp.com/album/dark-waters
Resolved: https://gabriellepietrangelo.bandcamp.com/album/big-desert-sky
Resolved: https://thefauxpaws.bandcamp.com/album/the-hurricane-ep
Resolved: https://fatherjohnmisty.bandcamp.com/album/anthem-3-2
Resolved: https://kellymcfarling.bandcamp.com/album/deep-the-habit
Resolved: https://joannanewsom.bandcamp.com/album/divers
Resolved: https://dogwood.bandcamp.com/album/the-imperfect-ep
Resolved: https://indigosparke.bandcamp.com/album/echo
Resolved: https://bedouine.bandcamp.com/album/bedouine
Resolved: https://imskullcrusher.bandcamp.com/album/skullcrusher
Resolved: https://fleetfoxes.bandcamp.com/album/crack-up
Resolved: https://craigmckerron.bandcamp.com/album/cabin-fever
Resolved: https://anothermichael.bandcamp.com/album/new-music-and-big-pop
Resolved: https://autourdelucie.bandcamp.com/album/bunker
Resolved: https://laurastevenson.bandcamp.com/album/sit-resist-remastered-deluxe-edition
Total 47 URLs resolved!
Now let's see what happens when we don't use limiter...
An error occurred: 429 Too Many Requests
Resolved: https://cassandrajenkins.bandcamp.com/album/an-overview-on-phenomenal-nature
Resolved: https://mrsgreenbird.bandcamp.com/album/10-years-live
Resolved: https://thefauxpaws.bandcamp.com/album/the-hurricane-ep
Resolved: https://mariabc.bandcamp.com/album/devils-rain-2
Resolved: https://fleetfoxes.bandcamp.com/album/crack-up
Resolved: https://ryleywalker.bandcamp.com/album/course-in-fable
Resolved: https://curtismcmurtry.bandcamp.com/album/toothless-messiah

View File

@ -1,4 +1,5 @@
const fetch = require('node-fetch');
const Bottleneck = require('bottleneck');
const utils = require('./utils.js');
const parser = require('./parser.js');
const Cache = require('./cache.js');
@ -414,7 +415,7 @@ async function _fetchPage(url, json = false, fetchOptions = null) {
else {
return json ? res.json() : res.text();
}
});
});
});
}
@ -435,7 +436,8 @@ const cache = {
clear: _cache.clear.bind(_cache)
};
module.exports = {
// Exported functions
const _exportFn = {
discover,
getDiscoverOptions,
sanitizeDiscoverParams,
@ -449,7 +451,6 @@ module.exports = {
search,
getAlbumHighlightsByTag,
getTags,
cache,
getAllShows,
getShow,
getArticleCategories,
@ -460,4 +461,18 @@ module.exports = {
getReleasesByTag,
searchTag,
searchLocation
};
};
// Bottleneck limiter
const _limiter = new Bottleneck({
maxConcurrent: 5,
minTime: 200
});
const limiter = {};
for (const [fnName, fn] of Object.entries(_exportFn)) {
limiter[fnName] = _limiter.wrap(fn);
}
limiter.updateSettings = _limiter.updateSettings.bind(_limiter);
// Module exports
module.exports = Object.assign({}, _exportFn, { cache, limiter });

View File

@ -22,6 +22,7 @@
"author": "Patrick Kan",
"license": "MIT",
"dependencies": {
"bottleneck": "^2.19.5",
"cheerio": "^1.0.0-rc.5",
"html-entities": "^2.0.2",
"node-cache": "^5.1.2",