maksimgrs98/bandcamp-fetch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement proper caching

Browse Source
This commit is contained in:
patrickkfkan 2021-01-26 15:58:39 +08:00
parent 5674fd1cc0
commit 0356531c4f
4 changed files with 151 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
lib/cache.js Normal file
View File

@ -0,0 +1,94 @@
const NodeCache = require('node-cache');
class Cache {
constructor(ttl, maxEntries) {
this._ttl = ttl;
this._maxEntries = maxEntries;
this._cache = new NodeCache({
checkperiod: 600
});
}
setTTL(type, ttl) {
const self = this;
if (self._ttl[type] != ttl[type]) {
self.getKeys(type).forEach( key => {
self._cache.ttl(key, ttl);
})
}
self._ttl = ttl;
}
setMaxEntries(type, maxEntries) {
this.reduceEntries(type, maxEntries);
this._maxEntries[type] = maxEntries;
}
getMaxEntries(type) {
return this._maxEntries[type] !== undefined ? this._maxEntries[type] : -1;
}
get(type, key) {
return this._cache.get(type + '.' + key);
}
put(type, key, value) {
const maxEntries = this.getMaxEntries();
if (maxEntries === 0) {
return false;
}
else if (maxEntries > 0) {
this.reduceEntries(maxEntries - 1);
}
return this._cache.set(type + '.' + key, value, this._ttl[type]);
}
reduceEntries(type, reduceTo) {
if (reduceTo === undefined) {
reduceTo = this.getMaxEntries(type);
}
if (reduceTo < 0) {
return;
}
const keys = this.getKeys(type);
if (keys.length > reduceTo) {
for (let i = 0; i < keys.length - reduceTo; i++) {
this._cache.del(keys[i]);
}
}
}
getKeys(type) {
return this._cache.keys().filter( key => key.startsWith(type + '.') );
}
clear(type) {
if (!type) {
this._cache.flushAll();
}
else {
this.getKeys(type).forEach( key => {
this._cache.del(key);
});
}
}
async getOrSet(type, key, promiseCallback) {
const self = this;
const cachedValue = self.get(type, key);
if (cachedValue !== undefined) {
return cachedValue;
}
else if (promiseCallback) {
return promiseCallback().then( value => {
self.put(type, key, value);
return value;
});
}
else {
return null;
}
}
}
module.exports = Cache;

121
lib/index.js
View File

@ -1,8 +1,9 @@
const fetch = require('node-fetch');
const utils = require('./utils.js');
const parser = require('./parser.js');
const Cache = require('./cache.js');
const cache = {};
const _cache = new Cache({ constant: 3600, page: 300 }, { page: 10 });
async function discover(params, options = {}) {
const imageConstants = await _getImageConstants();
@ -26,8 +27,7 @@ async function discover(params, options = {}) {
}
return utils.getDiscoverUrl(sanitizedParams);
})
.then( url => fetch(url) )
.then( res => res.json() )
.then( url => _fetchPage(url, true) )
.then( json => {
const result = parser.parseDiscoverResults(json, opts);
result.params = resultParams;
@ -80,16 +80,9 @@ async function sanitizeDiscoverParams(params) {
}
async function getDiscoverOptions() {
if (cache.discoverOptions !== undefined) {
return cache.discoverOptions;
}
const url = utils.getSiteUrl();
return fetch(url)
.then( res => res.text() )
.then( html => {
cache.discoverOptions = parser.parseDiscoverOptions(html);
return cache.discoverOptions;
});
return _cache.getOrSet('constant', 'discoverOptions', () => {
return _fetchPage(utils.getSiteUrl()).then( html => parser.parseDiscoverOptions(html) );
});
}
async function getImageFormat(idOrName) {
@ -121,16 +114,9 @@ async function getImageFormats(filter = '') {
}
async function _getImageConstants() {
if (cache.imageConstants !== undefined) {
return cache.imageConstants;
}
const url = utils.getSiteUrl();
return fetch(url)
.then( res => res.text() )
.then( html => {
cache.imageConstants = parser.parseImageConstants(html);
return cache.imageConstants;
});
return _cache.getOrSet('constant', 'imageConstants', () => {
return _fetchPage(utils.getSiteUrl()).then( html => parser.parseImageConstants(html) );
});
}
async function _parseImageFormatArg(arg, defaultId = null) {
@ -156,9 +142,7 @@ async function getAlbumInfo(albumUrl, options = {}) {
artistImageFormat: await _parseImageFormatArg(options.artistImageFormat),
includeRawData: options.includeRawData ? true : false
};
return fetch(albumUrl)
.then( res => res.text() )
.then( html => parser.parseAlbumInfo(html, opts) );
return _fetchPage(albumUrl).then( html => parser.parseAlbumInfo(html, opts) );
}
async function getTrackInfo(trackUrl, options = {}) {
@ -170,9 +154,7 @@ async function getTrackInfo(trackUrl, options = {}) {
artistImageFormat: await _parseImageFormatArg(options.artistImageFormat),
includeRawData: options.includeRawData ? true : false
};
return fetch(trackUrl)
.then( res => res.text() )
.then( html => parser.parseTrackInfo(html, opts) );
return _fetchPage(trackUrl).then( html => parser.parseTrackInfo(html, opts) );
}
async function getDiscography(artistOrLabelUrl, options = {}) {
@ -182,8 +164,7 @@ async function getDiscography(artistOrLabelUrl, options = {}) {
artistOrLabelUrl,
imageFormat: await _parseImageFormatArg(options.imageFormat, 9)
};
return fetch(utils.getUrl('music', artistOrLabelUrl))
.then( res => res.text() )
return _fetchPage(utils.getUrl('music', artistOrLabelUrl))
.then( html => parser.parseDiscography(html, opts) );
}
@ -197,30 +178,28 @@ async function getArtistOrLabelInfo(artistOrLabelUrl, options = {}) {
// 'music' page instead. For artists, if the 'music' page does not
// have the artist info, we shall try with an album or track page
// (this is inefficient...perhaps there is a better way?).
return fetch(utils.getUrl('music', artistOrLabelUrl))
.then( res => res.text() )
.then( html => parser.parseArtistOrLabelInfo(html, opts) )
.then( info => {
if (info.type === 'label' || info.name !== '') {
return info;
}
else {
return getDiscography(artistOrLabelUrl, options)
.then( discographyItems => {
const firstAlbumOrTrack = discographyItems[0];
if (firstAlbumOrTrack) {
return firstAlbumOrTrack.url;
}
else {
// fallback
return artistOrLabelUrl;
}
})
.then( url => fetch(url) )
.then( res => res.text() )
.then( html => parser.parseArtistOrLabelInfo(html, opts) );
}
});
return _fetchPage(utils.getUrl('music', artistOrLabelUrl))
.then( html => parser.parseArtistOrLabelInfo(html, opts) )
.then( info => {
if (info.type === 'label' || info.name !== '') {
return info;
}
else {
return getDiscography(artistOrLabelUrl, options)
.then( discographyItems => {
const firstAlbumOrTrack = discographyItems[0];
if (firstAlbumOrTrack) {
return firstAlbumOrTrack.url;
}
else {
// fallback
return artistOrLabelUrl;
}
})
.then( url => _fetchPage(url) )
.then( html => parser.parseArtistOrLabelInfo(html, opts) );
}
});
}
async function getLabelArtists(labelUrl, options = {}) {
@ -228,9 +207,8 @@ async function getLabelArtists(labelUrl, options = {}) {
labelUrl,
imageFormat: await _parseImageFormatArg(options.imageFormat)
};
return fetch(utils.getUrl('artists', labelUrl))
.then( res => res.text() )
.then( html => parser.parseLabelArtists(html, opts) );
return _fetchPage(utils.getUrl('artists', labelUrl))
.then( html => parser.parseLabelArtists(html, opts) );
}
async function search(params, options = {}) {
@ -238,8 +216,7 @@ async function search(params, options = {}) {
albumImageFormat: await _parseImageFormatArg(options.albumImageFormat),
artistImageFormat: await _parseImageFormatArg(options.artistImageFormat)
};
return fetch(utils.getSearchUrl(params))
.then( res => res.text() )
return _fetchPage(utils.getSearchUrl(params))
.then( html => parser.parseSearchResults(html, opts) );
}
@ -250,17 +227,30 @@ async function getAlbumHighlightsByTag(tagUrl, options = {}) {
imageFormat: await _parseImageFormatArg(options.imageFormat, 9)
};
return fetch(tagUrl)
.then( res => res.text() )
return _fetchPage(tagUrl)
.then( html => parser.parseAlbumHighlightsByTag(html, opts) );
}
async function getTags() {
return fetch(utils.getUrl('tags'))
.then( res => res.text() )
return _fetchPage(utils.getUrl('tags'))
.then( html => parser.parseTags(html) );
}
async function _fetchPage(url, json = false) {
return _cache.getOrSet('page', url + (json ? ':json' : ':html'), () => {
return fetch(url).then( res => json ? res.json() : res.text() );
});
}
// Cache functions
const cache = {
setTTL: _cache.setTTL.bind(_cache),
setMaxPages: (maxPages) => {
_cache.setMaxEntries('page', maxPages);
},
clear: _cache.clear.bind(_cache)
};
module.exports = {
discover,
getDiscoverOptions,
@ -274,5 +264,6 @@ module.exports = {
getLabelArtists,
search,
getAlbumHighlightsByTag,
getTags
getTags,
cache
};

1
lib/parser.js
View File

@ -264,7 +264,6 @@ function parseTrackInfo(html, opts) {
}
function parseTrackInfoFromAlbum(html, opts, trackPosition) {
console.log('parseTrackInfoFromAlbum: + ' + trackPosition);
const album = parseAlbumInfo(html, opts);
let trackData = album.tracks[trackPosition - 1] || {};
const track = {

1
package.json
View File

@ -24,6 +24,7 @@
"dependencies": {
"cheerio": "^1.0.0-rc.5",
"html-entities": "^2.0.2",
"node-cache": "^5.1.2",
"node-fetch": "^2.6.1"
}
}