From 0356531c4fdd5285da48b66ba80a9feeb7f15a11 Mon Sep 17 00:00:00 2001 From: patrickkfkan Date: Tue, 26 Jan 2021 15:58:39 +0800 Subject: [PATCH] Implement proper caching --- lib/cache.js | 94 +++++++++++++++++++++++++++++++++++++++ lib/index.js | 121 +++++++++++++++++++++++--------------------------- lib/parser.js | 1 - package.json | 1 + 4 files changed, 151 insertions(+), 66 deletions(-) create mode 100644 lib/cache.js diff --git a/lib/cache.js b/lib/cache.js new file mode 100644 index 0000000..05e760c --- /dev/null +++ b/lib/cache.js @@ -0,0 +1,94 @@ +const NodeCache = require('node-cache'); + +class Cache { + constructor(ttl, maxEntries) { + this._ttl = ttl; + this._maxEntries = maxEntries; + this._cache = new NodeCache({ + checkperiod: 600 + }); + } + + setTTL(type, ttl) { + const self = this; + if (self._ttl[type] != ttl[type]) { + self.getKeys(type).forEach( key => { + self._cache.ttl(key, ttl); + }) + } + self._ttl = ttl; + } + + setMaxEntries(type, maxEntries) { + this.reduceEntries(type, maxEntries); + this._maxEntries[type] = maxEntries; + } + + getMaxEntries(type) { + return this._maxEntries[type] !== undefined ? this._maxEntries[type] : -1; + } + + get(type, key) { + return this._cache.get(type + '.' + key); + } + + put(type, key, value) { + const maxEntries = this.getMaxEntries(); + if (maxEntries === 0) { + return false; + } + else if (maxEntries > 0) { + this.reduceEntries(maxEntries - 1); + } + return this._cache.set(type + '.' + key, value, this._ttl[type]); + } + + reduceEntries(type, reduceTo) { + if (reduceTo === undefined) { + reduceTo = this.getMaxEntries(type); + } + if (reduceTo < 0) { + return; + } + const keys = this.getKeys(type); + if (keys.length > reduceTo) { + for (let i = 0; i < keys.length - reduceTo; i++) { + this._cache.del(keys[i]); + } + } + } + + getKeys(type) { + return this._cache.keys().filter( key => key.startsWith(type + '.') ); + } + + clear(type) { + if (!type) { + this._cache.flushAll(); + } + else { + this.getKeys(type).forEach( key => { + this._cache.del(key); + }); + } + } + + async getOrSet(type, key, promiseCallback) { + const self = this; + const cachedValue = self.get(type, key); + if (cachedValue !== undefined) { + return cachedValue; + } + else if (promiseCallback) { + return promiseCallback().then( value => { + self.put(type, key, value); + return value; + }); + } + else { + return null; + } + } +} + +module.exports = Cache; \ No newline at end of file diff --git a/lib/index.js b/lib/index.js index ad29ba8..91de73c 100644 --- a/lib/index.js +++ b/lib/index.js @@ -1,8 +1,9 @@ const fetch = require('node-fetch'); const utils = require('./utils.js'); const parser = require('./parser.js'); +const Cache = require('./cache.js'); -const cache = {}; +const _cache = new Cache({ constant: 3600, page: 300 }, { page: 10 }); async function discover(params, options = {}) { const imageConstants = await _getImageConstants(); @@ -26,8 +27,7 @@ async function discover(params, options = {}) { } return utils.getDiscoverUrl(sanitizedParams); }) - .then( url => fetch(url) ) - .then( res => res.json() ) + .then( url => _fetchPage(url, true) ) .then( json => { const result = parser.parseDiscoverResults(json, opts); result.params = resultParams; @@ -80,16 +80,9 @@ async function sanitizeDiscoverParams(params) { } async function getDiscoverOptions() { - if (cache.discoverOptions !== undefined) { - return cache.discoverOptions; - } - const url = utils.getSiteUrl(); - return fetch(url) - .then( res => res.text() ) - .then( html => { - cache.discoverOptions = parser.parseDiscoverOptions(html); - return cache.discoverOptions; - }); + return _cache.getOrSet('constant', 'discoverOptions', () => { + return _fetchPage(utils.getSiteUrl()).then( html => parser.parseDiscoverOptions(html) ); + }); } async function getImageFormat(idOrName) { @@ -121,16 +114,9 @@ async function getImageFormats(filter = '') { } async function _getImageConstants() { - if (cache.imageConstants !== undefined) { - return cache.imageConstants; - } - const url = utils.getSiteUrl(); - return fetch(url) - .then( res => res.text() ) - .then( html => { - cache.imageConstants = parser.parseImageConstants(html); - return cache.imageConstants; - }); + return _cache.getOrSet('constant', 'imageConstants', () => { + return _fetchPage(utils.getSiteUrl()).then( html => parser.parseImageConstants(html) ); + }); } async function _parseImageFormatArg(arg, defaultId = null) { @@ -156,9 +142,7 @@ async function getAlbumInfo(albumUrl, options = {}) { artistImageFormat: await _parseImageFormatArg(options.artistImageFormat), includeRawData: options.includeRawData ? true : false }; - return fetch(albumUrl) - .then( res => res.text() ) - .then( html => parser.parseAlbumInfo(html, opts) ); + return _fetchPage(albumUrl).then( html => parser.parseAlbumInfo(html, opts) ); } async function getTrackInfo(trackUrl, options = {}) { @@ -170,9 +154,7 @@ async function getTrackInfo(trackUrl, options = {}) { artistImageFormat: await _parseImageFormatArg(options.artistImageFormat), includeRawData: options.includeRawData ? true : false }; - return fetch(trackUrl) - .then( res => res.text() ) - .then( html => parser.parseTrackInfo(html, opts) ); + return _fetchPage(trackUrl).then( html => parser.parseTrackInfo(html, opts) ); } async function getDiscography(artistOrLabelUrl, options = {}) { @@ -182,8 +164,7 @@ async function getDiscography(artistOrLabelUrl, options = {}) { artistOrLabelUrl, imageFormat: await _parseImageFormatArg(options.imageFormat, 9) }; - return fetch(utils.getUrl('music', artistOrLabelUrl)) - .then( res => res.text() ) + return _fetchPage(utils.getUrl('music', artistOrLabelUrl)) .then( html => parser.parseDiscography(html, opts) ); } @@ -197,30 +178,28 @@ async function getArtistOrLabelInfo(artistOrLabelUrl, options = {}) { // 'music' page instead. For artists, if the 'music' page does not // have the artist info, we shall try with an album or track page // (this is inefficient...perhaps there is a better way?). - return fetch(utils.getUrl('music', artistOrLabelUrl)) - .then( res => res.text() ) - .then( html => parser.parseArtistOrLabelInfo(html, opts) ) - .then( info => { - if (info.type === 'label' || info.name !== '') { - return info; - } - else { - return getDiscography(artistOrLabelUrl, options) - .then( discographyItems => { - const firstAlbumOrTrack = discographyItems[0]; - if (firstAlbumOrTrack) { - return firstAlbumOrTrack.url; - } - else { - // fallback - return artistOrLabelUrl; - } - }) - .then( url => fetch(url) ) - .then( res => res.text() ) - .then( html => parser.parseArtistOrLabelInfo(html, opts) ); - } - }); + return _fetchPage(utils.getUrl('music', artistOrLabelUrl)) + .then( html => parser.parseArtistOrLabelInfo(html, opts) ) + .then( info => { + if (info.type === 'label' || info.name !== '') { + return info; + } + else { + return getDiscography(artistOrLabelUrl, options) + .then( discographyItems => { + const firstAlbumOrTrack = discographyItems[0]; + if (firstAlbumOrTrack) { + return firstAlbumOrTrack.url; + } + else { + // fallback + return artistOrLabelUrl; + } + }) + .then( url => _fetchPage(url) ) + .then( html => parser.parseArtistOrLabelInfo(html, opts) ); + } + }); } async function getLabelArtists(labelUrl, options = {}) { @@ -228,9 +207,8 @@ async function getLabelArtists(labelUrl, options = {}) { labelUrl, imageFormat: await _parseImageFormatArg(options.imageFormat) }; - return fetch(utils.getUrl('artists', labelUrl)) - .then( res => res.text() ) - .then( html => parser.parseLabelArtists(html, opts) ); + return _fetchPage(utils.getUrl('artists', labelUrl)) + .then( html => parser.parseLabelArtists(html, opts) ); } async function search(params, options = {}) { @@ -238,8 +216,7 @@ async function search(params, options = {}) { albumImageFormat: await _parseImageFormatArg(options.albumImageFormat), artistImageFormat: await _parseImageFormatArg(options.artistImageFormat) }; - return fetch(utils.getSearchUrl(params)) - .then( res => res.text() ) + return _fetchPage(utils.getSearchUrl(params)) .then( html => parser.parseSearchResults(html, opts) ); } @@ -250,17 +227,30 @@ async function getAlbumHighlightsByTag(tagUrl, options = {}) { imageFormat: await _parseImageFormatArg(options.imageFormat, 9) }; - return fetch(tagUrl) - .then( res => res.text() ) + return _fetchPage(tagUrl) .then( html => parser.parseAlbumHighlightsByTag(html, opts) ); } async function getTags() { - return fetch(utils.getUrl('tags')) - .then( res => res.text() ) + return _fetchPage(utils.getUrl('tags')) .then( html => parser.parseTags(html) ); } +async function _fetchPage(url, json = false) { + return _cache.getOrSet('page', url + (json ? ':json' : ':html'), () => { + return fetch(url).then( res => json ? res.json() : res.text() ); + }); +} + +// Cache functions +const cache = { + setTTL: _cache.setTTL.bind(_cache), + setMaxPages: (maxPages) => { + _cache.setMaxEntries('page', maxPages); + }, + clear: _cache.clear.bind(_cache) +}; + module.exports = { discover, getDiscoverOptions, @@ -274,5 +264,6 @@ module.exports = { getLabelArtists, search, getAlbumHighlightsByTag, - getTags + getTags, + cache }; \ No newline at end of file diff --git a/lib/parser.js b/lib/parser.js index 79a37f2..028bdb4 100644 --- a/lib/parser.js +++ b/lib/parser.js @@ -264,7 +264,6 @@ function parseTrackInfo(html, opts) { } function parseTrackInfoFromAlbum(html, opts, trackPosition) { -console.log('parseTrackInfoFromAlbum: + ' + trackPosition); const album = parseAlbumInfo(html, opts); let trackData = album.tracks[trackPosition - 1] || {}; const track = { diff --git a/package.json b/package.json index fc74735..3a8c817 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "dependencies": { "cheerio": "^1.0.0-rc.5", "html-entities": "^2.0.2", + "node-cache": "^5.1.2", "node-fetch": "^2.6.1" } }