Implement proper caching

lib/cache.js

@@ -0,0 +1,94 @@
+const NodeCache = require('node-cache');
+
+class Cache {
+    constructor(ttl, maxEntries) {
+        this._ttl = ttl;
+        this._maxEntries = maxEntries;
+        this._cache = new NodeCache({
+            checkperiod: 600
+        });
+    }
+
+    setTTL(type, ttl) {
+        const self = this;
+        if (self._ttl[type] != ttl[type]) {
+            self.getKeys(type).forEach( key => {
+                self._cache.ttl(key, ttl);
+            })
+        }
+        self._ttl = ttl;
+    }
+
+    setMaxEntries(type, maxEntries) {
+        this.reduceEntries(type, maxEntries);
+        this._maxEntries[type] = maxEntries;
+    }
+
+    getMaxEntries(type) {
+        return this._maxEntries[type] !== undefined ? this._maxEntries[type] : -1;
+    }
+
+    get(type, key) {
+        return this._cache.get(type + '.' + key);
+    }
+
+    put(type, key, value) {
+        const maxEntries = this.getMaxEntries();
+        if (maxEntries === 0) {
+            return false;
+        }
+        else if (maxEntries > 0) {
+            this.reduceEntries(maxEntries - 1);
+        }
+        return this._cache.set(type + '.' + key, value, this._ttl[type]);
+    }
+
+    reduceEntries(type, reduceTo) {
+        if (reduceTo === undefined) {
+            reduceTo = this.getMaxEntries(type);
+        }
+        if (reduceTo < 0) {
+            return;
+        }
+        const keys = this.getKeys(type);
+        if (keys.length > reduceTo) {
+            for (let i = 0; i < keys.length - reduceTo; i++) {
+                this._cache.del(keys[i]);
+            }
+        }
+    }
+
+    getKeys(type) {
+        return this._cache.keys().filter( key => key.startsWith(type + '.') );
+    }
+
+    clear(type) {
+        if (!type) {
+            this._cache.flushAll();
+        }
+        else {
+            this.getKeys(type).forEach( key => {
+                this._cache.del(key);
+            });
+        }
+    }
+
+    async getOrSet(type, key, promiseCallback) {
+        const self = this;
+        const cachedValue = self.get(type, key);
+        if (cachedValue !== undefined) {
+            return cachedValue;
+        }
+        else if (promiseCallback) {
+            return promiseCallback().then( value => {
+                self.put(type, key, value);
+                return value;
+            });
+        }
+        else {
+            return null;
+        }
+    }
+}
+
+module.exports = Cache;

lib/index.js

@@ -1,8 +1,9 @@
 const fetch = require('node-fetch');
 const utils = require('./utils.js');
 const parser = require('./parser.js');
+const Cache = require('./cache.js');
 
-const cache = {};
+const _cache = new Cache({ constant: 3600, page: 300 }, { page: 10 });
 
 async function discover(params, options = {}) {
     const imageConstants = await _getImageConstants();
@@ -26,8 +27,7 @@ async function discover(params, options = {}) {
             }
             return utils.getDiscoverUrl(sanitizedParams);
         })
-        .then( url => fetch(url) )
+        .then( url => _fetchPage(url, true) )
-        .then( res => res.json() )
         .then( json => {
             const result = parser.parseDiscoverResults(json, opts);
             result.params = resultParams;
@@ -80,16 +80,9 @@ async function sanitizeDiscoverParams(params) {
 }
 
 async function getDiscoverOptions() {
-    if (cache.discoverOptions !== undefined) {
+    return _cache.getOrSet('constant', 'discoverOptions', () => {
-        return cache.discoverOptions;
+        return _fetchPage(utils.getSiteUrl()).then( html => parser.parseDiscoverOptions(html) );
-    }
+    });
-    const url = utils.getSiteUrl();
-    return fetch(url)
-        .then( res => res.text() )
-        .then( html => {
-            cache.discoverOptions = parser.parseDiscoverOptions(html);
-            return cache.discoverOptions;
-        });
 }
 
 async function getImageFormat(idOrName) {
@@ -121,16 +114,9 @@ async function getImageFormats(filter = '') {
 }
 
 async function _getImageConstants() {
-    if (cache.imageConstants !== undefined) {
+    return _cache.getOrSet('constant', 'imageConstants', () => {
-        return cache.imageConstants;
+        return _fetchPage(utils.getSiteUrl()).then( html => parser.parseImageConstants(html) );
-    }
+    });
-    const url = utils.getSiteUrl();
-    return fetch(url)
-        .then( res => res.text() )
-        .then( html => {
-            cache.imageConstants = parser.parseImageConstants(html);
-            return cache.imageConstants;
-        });
 }
 
 async function _parseImageFormatArg(arg, defaultId = null) {
@@ -156,9 +142,7 @@ async function getAlbumInfo(albumUrl, options = {}) {
         artistImageFormat: await _parseImageFormatArg(options.artistImageFormat),
         includeRawData: options.includeRawData ? true : false
     };
-    return fetch(albumUrl)
+    return _fetchPage(albumUrl).then( html => parser.parseAlbumInfo(html, opts) );
-        .then( res => res.text() )
-        .then( html => parser.parseAlbumInfo(html, opts) );
 }
 
 async function getTrackInfo(trackUrl, options = {}) {
@@ -170,9 +154,7 @@ async function getTrackInfo(trackUrl, options = {}) {
         artistImageFormat: await _parseImageFormatArg(options.artistImageFormat),
         includeRawData: options.includeRawData ? true : false
     };
-    return fetch(trackUrl)
+    return _fetchPage(trackUrl).then( html => parser.parseTrackInfo(html, opts) );
-        .then( res => res.text() )
-        .then( html => parser.parseTrackInfo(html, opts) );
 }
 
 async function getDiscography(artistOrLabelUrl, options = {}) {
@@ -182,8 +164,7 @@ async function getDiscography(artistOrLabelUrl, options = {}) {
         artistOrLabelUrl,
         imageFormat: await _parseImageFormatArg(options.imageFormat, 9)
     };
-    return fetch(utils.getUrl('music', artistOrLabelUrl))
+    return _fetchPage(utils.getUrl('music', artistOrLabelUrl))
-        .then( res => res.text() )
         .then( html => parser.parseDiscography(html, opts) );
 }
 
@@ -197,30 +178,28 @@ async function getArtistOrLabelInfo(artistOrLabelUrl, options = {}) {
     // 'music' page instead. For artists, if the 'music' page does not
     // have the artist info, we shall try with an album or track page
     // (this is inefficient...perhaps there is a better way?).
-    return fetch(utils.getUrl('music', artistOrLabelUrl))
+    return _fetchPage(utils.getUrl('music', artistOrLabelUrl))
-        .then( res => res.text() )
+            .then( html => parser.parseArtistOrLabelInfo(html, opts) )
-        .then( html => parser.parseArtistOrLabelInfo(html, opts) )
+            .then( info => {
-        .then( info => {
+                if (info.type === 'label' || info.name !== '') {
-            if (info.type === 'label' || info.name !== '') {
+                    return info;
-                return info;
+                }
-            }
+                else {
-            else {
+                    return getDiscography(artistOrLabelUrl, options)
-                return getDiscography(artistOrLabelUrl, options)
+                        .then( discographyItems => {
-                    .then( discographyItems => {
+                            const firstAlbumOrTrack = discographyItems[0];
-                        const firstAlbumOrTrack = discographyItems[0];
+                            if (firstAlbumOrTrack) {
-                        if (firstAlbumOrTrack) {
+                                return firstAlbumOrTrack.url;
-                            return firstAlbumOrTrack.url;
+                            }
-                        }
+                            else {
-                        else {
+                                // fallback
-                            // fallback
+                                return artistOrLabelUrl;
-                            return artistOrLabelUrl;
+                            }
-                        }
+                        })
-                    })
+                        .then( url => _fetchPage(url) )
-                    .then( url => fetch(url) )
+                        .then( html => parser.parseArtistOrLabelInfo(html, opts) );
-                    .then( res => res.text() )
+                }
-                    .then( html => parser.parseArtistOrLabelInfo(html, opts) );
+            });
-            }
-        });
 }
 
 async function getLabelArtists(labelUrl, options = {}) {
@@ -228,8 +207,7 @@ async function getLabelArtists(labelUrl, options = {}) {
         labelUrl,
         imageFormat: await _parseImageFormatArg(options.imageFormat)
     };
-    return fetch(utils.getUrl('artists', labelUrl))
+    return _fetchPage(utils.getUrl('artists', labelUrl))
-        .then( res => res.text() )
         .then( html => parser.parseLabelArtists(html, opts) );        
 }
 
@@ -238,8 +216,7 @@ async function search(params, options = {}) {
         albumImageFormat: await _parseImageFormatArg(options.albumImageFormat),
         artistImageFormat: await _parseImageFormatArg(options.artistImageFormat)
     };
-    return fetch(utils.getSearchUrl(params))
+    return _fetchPage(utils.getSearchUrl(params))
-        .then( res => res.text() )
         .then( html => parser.parseSearchResults(html, opts) );
 }
 
@@ -250,17 +227,30 @@ async function getAlbumHighlightsByTag(tagUrl, options = {}) {
         imageFormat: await _parseImageFormatArg(options.imageFormat, 9)
     };
 
-    return fetch(tagUrl)
+    return _fetchPage(tagUrl)
-        .then( res => res.text() )
         .then( html => parser.parseAlbumHighlightsByTag(html, opts) );  
 }
 
 async function getTags() {
-    return fetch(utils.getUrl('tags'))
+    return _fetchPage(utils.getUrl('tags'))
-        .then( res => res.text() )
         .then( html => parser.parseTags(html) );  
 }
 
+async function _fetchPage(url, json = false) {
+    return _cache.getOrSet('page', url + (json ? ':json' : ':html'), () => {
+        return fetch(url).then( res => json ? res.json() : res.text() );
+    });
+}
+
+// Cache functions
+const cache = {
+    setTTL: _cache.setTTL.bind(_cache),
+    setMaxPages: (maxPages) => {
+        _cache.setMaxEntries('page', maxPages);
+    },
+    clear: _cache.clear.bind(_cache)
+};
+
 module.exports = {
     discover,
     getDiscoverOptions,
@@ -274,5 +264,6 @@ module.exports = {
     getLabelArtists,
     search,
     getAlbumHighlightsByTag,
-    getTags
+    getTags,
+    cache
 };

lib/parser.js

@@ -264,7 +264,6 @@ function parseTrackInfo(html, opts) {
 }
 
 function parseTrackInfoFromAlbum(html, opts, trackPosition) {
-console.log('parseTrackInfoFromAlbum: + ' + trackPosition);
     const album = parseAlbumInfo(html, opts);
     let trackData = album.tracks[trackPosition - 1] || {};
     const track = {

package.json

@@ -24,6 +24,7 @@
   "dependencies": {
     "cheerio": "^1.0.0-rc.5",
     "html-entities": "^2.0.2",
+    "node-cache": "^5.1.2",
     "node-fetch": "^2.6.1"
   }
 }