diff options
Diffstat (limited to 'node_modules/cacheable-request')
| -rw-r--r-- | node_modules/cacheable-request/LICENSE | 21 | ||||
| -rw-r--r-- | node_modules/cacheable-request/README.md | 206 | ||||
| -rw-r--r-- | node_modules/cacheable-request/package.json | 56 | ||||
| -rw-r--r-- | node_modules/cacheable-request/src/index.js | 251 |
4 files changed, 534 insertions, 0 deletions
diff --git a/node_modules/cacheable-request/LICENSE b/node_modules/cacheable-request/LICENSE new file mode 100644 index 0000000..f27ee9b --- /dev/null +++ b/node_modules/cacheable-request/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Luke Childs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/node_modules/cacheable-request/README.md b/node_modules/cacheable-request/README.md new file mode 100644 index 0000000..725e7e0 --- /dev/null +++ b/node_modules/cacheable-request/README.md @@ -0,0 +1,206 @@ +# cacheable-request + +> Wrap native HTTP requests with RFC compliant cache support + +[](https://travis-ci.org/lukechilds/cacheable-request) +[](https://coveralls.io/github/lukechilds/cacheable-request?branch=master) +[](https://www.npmjs.com/package/cacheable-request) +[](https://www.npmjs.com/package/cacheable-request) + +[RFC 7234](http://httpwg.org/specs/rfc7234.html) compliant HTTP caching for native Node.js HTTP/HTTPS requests. Caching works out of the box in memory or is easily pluggable with a wide range of storage adapters. + +**Note:** This is a low level wrapper around the core HTTP modules, it's not a high level request library. + +## Features + +- Only stores cacheable responses as defined by RFC 7234 +- Fresh cache entries are served directly from cache +- Stale cache entries are revalidated with `If-None-Match`/`If-Modified-Since` headers +- 304 responses from revalidation requests use cached body +- Updates `Age` header on cached responses +- Can completely bypass cache on a per request basis +- In memory cache by default +- Official support for Redis, MongoDB, SQLite, PostgreSQL and MySQL storage adapters +- Easily plug in your own or third-party storage adapters +- If DB connection fails, cache is automatically bypassed ([disabled by default](#optsautomaticfailover)) +- Adds cache support to any existing HTTP code with minimal changes +- Uses [http-cache-semantics](https://github.com/pornel/http-cache-semantics) internally for HTTP RFC 7234 compliance + +## Install + +```shell +npm install cacheable-request +``` + +## Usage + +```js +const http = require('http'); +const CacheableRequest = require('cacheable-request'); + +// Then instead of +const req = http.request('http://example.com', cb); +req.end(); + +// You can do +const cacheableRequest = new CacheableRequest(http.request); +const cacheReq = cacheableRequest('http://example.com', cb); +cacheReq.on('request', req => req.end()); +// Future requests to 'example.com' will be returned from cache if still valid + +// You pass in any other http.request API compatible method to be wrapped with cache support: +const cacheableRequest = new CacheableRequest(https.request); +const cacheableRequest = new CacheableRequest(electron.net); +``` + +## Storage Adapters + +`cacheable-request` uses [Keyv](https://github.com/lukechilds/keyv) to support a wide range of storage adapters. + +For example, to use Redis as a cache backend, you just need to install the official Redis Keyv storage adapter: + +``` +npm install @keyv/redis +``` + +And then you can pass `CacheableRequest` your connection string: + +```js +const cacheableRequest = new CacheableRequest(http.request, 'redis://user:pass@localhost:6379'); +``` + +[View all official Keyv storage adapters.](https://github.com/lukechilds/keyv#official-storage-adapters) + +Keyv also supports anything that follows the Map API so it's easy to write your own storage adapter or use a third-party solution. + +e.g The following are all valid storage adapters + +```js +const storageAdapter = new Map(); +// or +const storageAdapter = require('./my-storage-adapter'); +// or +const QuickLRU = require('quick-lru'); +const storageAdapter = new QuickLRU({ maxSize: 1000 }); + +const cacheableRequest = new CacheableRequest(http.request, storageAdapter); +``` + +View the [Keyv docs](https://github.com/lukechilds/keyv) for more information on how to use storage adapters. + +## API + +### new cacheableRequest(request, [storageAdapter]) + +Returns the provided request function wrapped with cache support. + +#### request + +Type: `function` + +Request function to wrap with cache support. Should be [`http.request`](https://nodejs.org/api/http.html#http_http_request_options_callback) or a similar API compatible request function. + +#### storageAdapter + +Type: `Keyv storage adapter`<br> +Default: `new Map()` + +A [Keyv](https://github.com/lukechilds/keyv) storage adapter instance, or connection string if using with an official Keyv storage adapter. + +### Instance + +#### cacheableRequest(opts, [cb]) + +Returns an event emitter. + +##### opts + +Type: `object`, `string` + +- Any of the default request functions options. +- Any [`http-cache-semantics`](https://github.com/kornelski/http-cache-semantics#constructor-options) options. +- Any of the following: + +###### opts.cache + +Type: `boolean`<br> +Default: `true` + +If the cache should be used. Setting this to false will completely bypass the cache for the current request. + +###### opts.strictTtl + +Type: `boolean`<br> +Default: `false` + +If set to `true` once a cached resource has expired it is deleted and will have to be re-requested. + +If set to `false` (default), after a cached resource's TTL expires it is kept in the cache and will be revalidated on the next request with `If-None-Match`/`If-Modified-Since` headers. + +###### opts.maxTtl + +Type: `number`<br> +Default: `undefined` + +Limits TTL. The `number` represents milliseconds. + +###### opts.automaticFailover + +Type: `boolean`<br> +Default: `false` + +When set to `true`, if the DB connection fails we will automatically fallback to a network request. DB errors will still be emitted to notify you of the problem even though the request callback may succeed. + +###### opts.forceRefresh + +Type: `boolean`<br> +Default: `false` + +Forces refreshing the cache. If the response could be retrieved from the cache, it will perform a new request and override the cache instead. + +##### cb + +Type: `function` + +The callback function which will receive the response as an argument. + +The response can be either a [Node.js HTTP response stream](https://nodejs.org/api/http.html#http_class_http_incomingmessage) or a [responselike object](https://github.com/lukechilds/responselike). The response will also have a `fromCache` property set with a boolean value. + +##### .on('request', request) + +`request` event to get the request object of the request. + +**Note:** This event will only fire if an HTTP request is actually made, not when a response is retrieved from cache. However, you should always handle the `request` event to end the request and handle any potential request errors. + +##### .on('response', response) + +`response` event to get the response object from the HTTP request or cache. + +##### .on('error', error) + +`error` event emitted in case of an error with the cache. + +Errors emitted here will be an instance of `CacheableRequest.RequestError` or `CacheableRequest.CacheError`. You will only ever receive a `RequestError` if the request function throws (normally caused by invalid user input). Normal request errors should be handled inside the `request` event. + +To properly handle all error scenarios you should use the following pattern: + +```js +cacheableRequest('example.com', cb) + .on('error', err => { + if (err instanceof CacheableRequest.CacheError) { + handleCacheError(err); // Cache error + } else if (err instanceof CacheableRequest.RequestError) { + handleRequestError(err); // Request function thrown + } + }) + .on('request', req => { + req.on('error', handleRequestError); // Request error emitted + req.end(); + }); +``` + +**Note:** Database connection errors are emitted here, however `cacheable-request` will attempt to re-request the resource and bypass the cache on a connection error. Therefore a database connection error doesn't necessarily mean the request won't be fulfilled. + +## License + +MIT © Luke Childs diff --git a/node_modules/cacheable-request/package.json b/node_modules/cacheable-request/package.json new file mode 100644 index 0000000..b3e0242 --- /dev/null +++ b/node_modules/cacheable-request/package.json @@ -0,0 +1,56 @@ +{ + "name": "cacheable-request", + "version": "7.0.2", + "description": "Wrap native HTTP requests with RFC compliant cache support", + "license": "MIT", + "repository": "lukechilds/cacheable-request", + "author": "Luke Childs <lukechilds123@gmail.com> (http://lukechilds.co.uk)", + "main": "src/index.js", + "engines": { + "node": ">=8" + }, + "scripts": { + "test": "xo && nyc ava", + "coverage": "nyc report --reporter=text-lcov | coveralls" + }, + "files": [ + "src" + ], + "keywords": [ + "HTTP", + "HTTPS", + "cache", + "caching", + "layer", + "cacheable", + "RFC 7234", + "RFC", + "7234", + "compliant" + ], + "dependencies": { + "clone-response": "^1.0.2", + "get-stream": "^5.1.0", + "http-cache-semantics": "^4.0.0", + "keyv": "^4.0.0", + "lowercase-keys": "^2.0.0", + "normalize-url": "^6.0.1", + "responselike": "^2.0.0" + }, + "devDependencies": { + "@keyv/sqlite": "^2.0.0", + "ava": "^1.1.0", + "coveralls": "^3.0.0", + "create-test-server": "3.0.0", + "delay": "^4.0.0", + "eslint-config-xo-lukechilds": "^1.0.0", + "nyc": "^14.1.1", + "pify": "^4.0.0", + "sqlite3": "^4.0.2", + "this": "^1.0.2", + "xo": "^0.23.0" + }, + "xo": { + "extends": "xo-lukechilds" + } +} diff --git a/node_modules/cacheable-request/src/index.js b/node_modules/cacheable-request/src/index.js new file mode 100644 index 0000000..3fcea3f --- /dev/null +++ b/node_modules/cacheable-request/src/index.js @@ -0,0 +1,251 @@ +'use strict'; + +const EventEmitter = require('events'); +const urlLib = require('url'); +const normalizeUrl = require('normalize-url'); +const getStream = require('get-stream'); +const CachePolicy = require('http-cache-semantics'); +const Response = require('responselike'); +const lowercaseKeys = require('lowercase-keys'); +const cloneResponse = require('clone-response'); +const Keyv = require('keyv'); + +class CacheableRequest { + constructor(request, cacheAdapter) { + if (typeof request !== 'function') { + throw new TypeError('Parameter `request` must be a function'); + } + + this.cache = new Keyv({ + uri: typeof cacheAdapter === 'string' && cacheAdapter, + store: typeof cacheAdapter !== 'string' && cacheAdapter, + namespace: 'cacheable-request' + }); + + return this.createCacheableRequest(request); + } + + createCacheableRequest(request) { + return (opts, cb) => { + let url; + if (typeof opts === 'string') { + url = normalizeUrlObject(urlLib.parse(opts)); + opts = {}; + } else if (opts instanceof urlLib.URL) { + url = normalizeUrlObject(urlLib.parse(opts.toString())); + opts = {}; + } else { + const [pathname, ...searchParts] = (opts.path || '').split('?'); + const search = searchParts.length > 0 ? + `?${searchParts.join('?')}` : + ''; + url = normalizeUrlObject({ ...opts, pathname, search }); + } + + opts = { + headers: {}, + method: 'GET', + cache: true, + strictTtl: false, + automaticFailover: false, + ...opts, + ...urlObjectToRequestOptions(url) + }; + opts.headers = lowercaseKeys(opts.headers); + + const ee = new EventEmitter(); + const normalizedUrlString = normalizeUrl( + urlLib.format(url), + { + stripWWW: false, + removeTrailingSlash: false, + stripAuthentication: false + } + ); + const key = `${opts.method}:${normalizedUrlString}`; + let revalidate = false; + let madeRequest = false; + + const makeRequest = opts => { + madeRequest = true; + let requestErrored = false; + let requestErrorCallback; + + const requestErrorPromise = new Promise(resolve => { + requestErrorCallback = () => { + if (!requestErrored) { + requestErrored = true; + resolve(); + } + }; + }); + + const handler = response => { + if (revalidate && !opts.forceRefresh) { + response.status = response.statusCode; + const revalidatedPolicy = CachePolicy.fromObject(revalidate.cachePolicy).revalidatedPolicy(opts, response); + if (!revalidatedPolicy.modified) { + const headers = revalidatedPolicy.policy.responseHeaders(); + response = new Response(revalidate.statusCode, headers, revalidate.body, revalidate.url); + response.cachePolicy = revalidatedPolicy.policy; + response.fromCache = true; + } + } + + if (!response.fromCache) { + response.cachePolicy = new CachePolicy(opts, response, opts); + response.fromCache = false; + } + + let clonedResponse; + if (opts.cache && response.cachePolicy.storable()) { + clonedResponse = cloneResponse(response); + + (async () => { + try { + const bodyPromise = getStream.buffer(response); + + await Promise.race([ + requestErrorPromise, + new Promise(resolve => response.once('end', resolve)) + ]); + + if (requestErrored) { + return; + } + + const body = await bodyPromise; + + const value = { + cachePolicy: response.cachePolicy.toObject(), + url: response.url, + statusCode: response.fromCache ? revalidate.statusCode : response.statusCode, + body + }; + + let ttl = opts.strictTtl ? response.cachePolicy.timeToLive() : undefined; + if (opts.maxTtl) { + ttl = ttl ? Math.min(ttl, opts.maxTtl) : opts.maxTtl; + } + + await this.cache.set(key, value, ttl); + } catch (error) { + ee.emit('error', new CacheableRequest.CacheError(error)); + } + })(); + } else if (opts.cache && revalidate) { + (async () => { + try { + await this.cache.delete(key); + } catch (error) { + ee.emit('error', new CacheableRequest.CacheError(error)); + } + })(); + } + + ee.emit('response', clonedResponse || response); + if (typeof cb === 'function') { + cb(clonedResponse || response); + } + }; + + try { + const req = request(opts, handler); + req.once('error', requestErrorCallback); + req.once('abort', requestErrorCallback); + ee.emit('request', req); + } catch (error) { + ee.emit('error', new CacheableRequest.RequestError(error)); + } + }; + + (async () => { + const get = async opts => { + await Promise.resolve(); + + const cacheEntry = opts.cache ? await this.cache.get(key) : undefined; + if (typeof cacheEntry === 'undefined') { + return makeRequest(opts); + } + + const policy = CachePolicy.fromObject(cacheEntry.cachePolicy); + if (policy.satisfiesWithoutRevalidation(opts) && !opts.forceRefresh) { + const headers = policy.responseHeaders(); + const response = new Response(cacheEntry.statusCode, headers, cacheEntry.body, cacheEntry.url); + response.cachePolicy = policy; + response.fromCache = true; + + ee.emit('response', response); + if (typeof cb === 'function') { + cb(response); + } + } else { + revalidate = cacheEntry; + opts.headers = policy.revalidationHeaders(opts); + makeRequest(opts); + } + }; + + const errorHandler = error => ee.emit('error', new CacheableRequest.CacheError(error)); + this.cache.once('error', errorHandler); + ee.on('response', () => this.cache.removeListener('error', errorHandler)); + + try { + await get(opts); + } catch (error) { + if (opts.automaticFailover && !madeRequest) { + makeRequest(opts); + } + + ee.emit('error', new CacheableRequest.CacheError(error)); + } + })(); + + return ee; + }; + } +} + +function urlObjectToRequestOptions(url) { + const options = { ...url }; + options.path = `${url.pathname || '/'}${url.search || ''}`; + delete options.pathname; + delete options.search; + return options; +} + +function normalizeUrlObject(url) { + // If url was parsed by url.parse or new URL: + // - hostname will be set + // - host will be hostname[:port] + // - port will be set if it was explicit in the parsed string + // Otherwise, url was from request options: + // - hostname or host may be set + // - host shall not have port encoded + return { + protocol: url.protocol, + auth: url.auth, + hostname: url.hostname || url.host || 'localhost', + port: url.port, + pathname: url.pathname, + search: url.search + }; +} + +CacheableRequest.RequestError = class extends Error { + constructor(error) { + super(error.message); + this.name = 'RequestError'; + Object.assign(this, error); + } +}; + +CacheableRequest.CacheError = class extends Error { + constructor(error) { + super(error.message); + this.name = 'CacheError'; + Object.assign(this, error); + } +}; + +module.exports = CacheableRequest; |