From 288d196cf12639b0760da5b5a07257a95d4d9321 Mon Sep 17 00:00:00 2001 From: Guido Lorenz Date: Tue, 23 Feb 2021 16:38:45 +0100 Subject: [PATCH 1/9] Fix usage of Buffers as values --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 0f3d39f..bd5b3f4 100644 --- a/index.js +++ b/index.js @@ -138,7 +138,7 @@ RedisDown.prototype._put = function (key, rawvalue, opt, callback) { if (typeof rawvalue === 'undefined' || rawvalue === null) { rawvalue = ''; } - this.__exec(this.__appendPutCmd([], key, rawvalue.toString()), callback); + this.__exec(this.__appendPutCmd([], key, cleanKey(rawvalue)), callback); }; RedisDown.prototype._del = function (key, opt, cb) { From b61bb3dd4a9ff28a163648b54e0770f281d1df65 Mon Sep 17 00:00:00 2001 From: Guido Lorenz Date: Tue, 23 Feb 2021 16:39:21 +0100 Subject: [PATCH 2/9] Rename cleanKey() function --- index.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/index.js b/index.js index bd5b3f4..091e8f5 100644 --- a/index.js +++ b/index.js @@ -111,7 +111,7 @@ RedisDown.prototype._open = function (options, callback) { }; RedisDown.prototype._get = function (key, options, callback) { - this.db.hget(this.location + ':h', cleanKey(key), function (e, v) { + this.db.hget(this.location + ':h', toBufferOrString(key), function (e, v) { if (e) { return setImmediate(function callNext() { return callback(e); @@ -138,7 +138,7 @@ RedisDown.prototype._put = function (key, rawvalue, opt, callback) { if (typeof rawvalue === 'undefined' || rawvalue === null) { rawvalue = ''; } - this.__exec(this.__appendPutCmd([], key, cleanKey(rawvalue)), callback); + this.__exec(this.__appendPutCmd([], key, toBufferOrString(rawvalue)), callback); }; RedisDown.prototype._del = function (key, opt, cb) { @@ -166,14 +166,14 @@ RedisDown.prototype.__getPrefix = function (prefix) { RedisDown.prototype.__appendPutCmd = function (commandList, key, value, prefix) { var resolvedPrefix = this.__getPrefix(prefix); - key = cleanKey(key); + key = toBufferOrString(key); commandList.push(['hset', resolvedPrefix + ':h', key, value === undefined ? '' : value]); commandList.push(['zadd', resolvedPrefix + ':z', 0, key]); return commandList; }; RedisDown.prototype.__appendDelCmd = function (commandList, key, prefix) { - key = cleanKey(key); + key = toBufferOrString(key); var resolvedPrefix = this.__getPrefix(prefix); commandList.push(['hdel', resolvedPrefix + ':h', key]); commandList.push(['zrem', resolvedPrefix + ':z', key]); @@ -300,7 +300,7 @@ function sanitizeLocation(location) { return location; } -function cleanKey(key) { +function toBufferOrString(key) { if (Buffer.isBuffer(key)) { return key; } else { From 5edb32891c657330bfde0302b5dedaf9b2023968 Mon Sep 17 00:00:00 2001 From: Guido Lorenz Date: Tue, 23 Feb 2021 16:47:21 +0100 Subject: [PATCH 3/9] Store values as individual keys to allow partial cache eviction --- index.js | 10 +++++++--- luascripts/zhpairs.lua | 8 ++++++-- luascripts/zhrevpairs.lua | 6 +++++- luascripts/zhrevvalues-lk.lua | 6 +++++- luascripts/zhvalues-lk.lua | 6 +++++- 5 files changed, 28 insertions(+), 8 deletions(-) diff --git a/index.js b/index.js index 091e8f5..7ed3418 100644 --- a/index.js +++ b/index.js @@ -111,7 +111,7 @@ RedisDown.prototype._open = function (options, callback) { }; RedisDown.prototype._get = function (key, options, callback) { - this.db.hget(this.location + ':h', toBufferOrString(key), function (e, v) { + this.db.get(valueKey(this.location, key), function (e, v) { if (e) { return setImmediate(function callNext() { return callback(e); @@ -167,7 +167,7 @@ RedisDown.prototype.__getPrefix = function (prefix) { RedisDown.prototype.__appendPutCmd = function (commandList, key, value, prefix) { var resolvedPrefix = this.__getPrefix(prefix); key = toBufferOrString(key); - commandList.push(['hset', resolvedPrefix + ':h', key, value === undefined ? '' : value]); + commandList.push(['set', valueKey(resolvedPrefix, key), value === undefined ? '' : value]); commandList.push(['zadd', resolvedPrefix + ':z', 0, key]); return commandList; }; @@ -175,7 +175,7 @@ RedisDown.prototype.__appendPutCmd = function (commandList, key, value, prefix) RedisDown.prototype.__appendDelCmd = function (commandList, key, prefix) { key = toBufferOrString(key); var resolvedPrefix = this.__getPrefix(prefix); - commandList.push(['hdel', resolvedPrefix + ':h', key]); + commandList.push(['del', valueKey(resolvedPrefix, key)]); commandList.push(['zrem', resolvedPrefix + ':z', key]); return commandList; }; @@ -308,6 +308,10 @@ function toBufferOrString(key) { } } +function valueKey(location, key) { + return location + '$' + key; +} + RedisDown.reset = function (callback) { for (var k in RedisDown.dbs) { if (RedisDown.dbs.hasOwnProperty(k)) { diff --git a/luascripts/zhpairs.lua b/luascripts/zhpairs.lua index a6c18ec..9291d75 100644 --- a/luascripts/zhpairs.lua +++ b/luascripts/zhpairs.lua @@ -9,10 +9,14 @@ local keys = redis.call('zrangebylex', KEYS[1]..':z', unpack(ARGV)) if #keys == 0 then return keys end -local values = redis.call('hmget',KEYS[1]..':h',unpack(keys)) +local prefixkeys = {} +for i,v in ipairs(keys) do + prefixkeys[i] = KEYS[1]..'$'..v +end +local values = redis.call('mget', unpack(prefixkeys)) local result = {} for i,v in ipairs(keys) do result[i*2-1] = v result[i*2] = values[i] end -return result \ No newline at end of file +return result diff --git a/luascripts/zhrevpairs.lua b/luascripts/zhrevpairs.lua index 7b5cf0a..eb7526a 100644 --- a/luascripts/zhrevpairs.lua +++ b/luascripts/zhrevpairs.lua @@ -9,7 +9,11 @@ local keys = redis.call('zrevrangebylex', KEYS[1]..':z', unpack(ARGV)) if #keys == 0 then return keys end -local values = redis.call('hmget',KEYS[1]..':h',unpack(keys)) +local prefixkeys = {} +for i,v in ipairs(keys) do + prefixkeys[i] = KEYS[1]..'$'..v +end +local values = redis.call('mget', unpack(prefixkeys)) local result = {} for i,v in ipairs(keys) do result[i*2-1] = v diff --git a/luascripts/zhrevvalues-lk.lua b/luascripts/zhrevvalues-lk.lua index f5950b7..f0c2e9a 100644 --- a/luascripts/zhrevvalues-lk.lua +++ b/luascripts/zhrevvalues-lk.lua @@ -10,6 +10,10 @@ local keys = redis.call('zrevrangebylex', KEYS[1]..':z', unpack(ARGV)) if #keys == 0 then return keys end -local values = redis.call('hmget', KEYS[1]..':h', unpack(keys)) +local prefixkeys = {} +for i,v in ipairs(keys) do + prefixkeys[i] = KEYS[1]..'$'..v +end +local values = redis.call('mget', unpack(prefixkeys)) values[#values+1] = keys[#keys] return values diff --git a/luascripts/zhvalues-lk.lua b/luascripts/zhvalues-lk.lua index 01d9463..0a7cd81 100644 --- a/luascripts/zhvalues-lk.lua +++ b/luascripts/zhvalues-lk.lua @@ -10,6 +10,10 @@ local keys = redis.call('zrangebylex', KEYS[1]..':z', unpack(ARGV)) if #keys == 0 then return keys end -local values = redis.call('hmget', KEYS[1]..':h', unpack(keys)) +local prefixkeys = {} +for i,v in ipairs(keys) do + prefixkeys[i] = KEYS[1]..'$'..v +end +local values = redis.call('mget', unpack(prefixkeys)) values[#values+1] = keys[#keys] return values From bb7e246423c588eafc8873d39a033677bed45b41 Mon Sep 17 00:00:00 2001 From: Guido Lorenz Date: Tue, 23 Feb 2021 16:49:16 +0100 Subject: [PATCH 4/9] Allow passing tls and password options down to redis --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 7ed3418..c9965fb 100644 --- a/index.js +++ b/index.js @@ -261,7 +261,7 @@ RedisDown.prototype.destroy = function (doClose, callback) { * when the identifier is identical, it is safe to reuse the same client. */ function _makeRedisId(location, options) { - var redisIdOptions = ['host', 'port', + var redisIdOptions = ['host', 'port', 'tls', 'password', 'parser', 'return_buffers', 'detect_buffers', 'socket_nodelay', 'no_ready_check', 'enable_offline_queue', 'retry_max_delay', 'connect_timeout', 'max_attempts' ]; From 96e8f9e95ff28a03951afd2216e881eeccf11b9e Mon Sep 17 00:00:00 2001 From: Guido Lorenz Date: Tue, 23 Feb 2021 16:49:57 +0100 Subject: [PATCH 5/9] Delete TravisCI config --- .travis.yml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index ed55509..0000000 --- a/.travis.yml +++ /dev/null @@ -1,5 +0,0 @@ -language: node_js -node_js: - - "12.16.1" -services: - - redis-server From adf5d66e7b5e3e04902f1cc29425dc96cc1f0081 Mon Sep 17 00:00:00 2001 From: Guido Lorenz Date: Tue, 23 Feb 2021 16:56:14 +0100 Subject: [PATCH 6/9] Rename package and update README --- LICENSE | 1 + README.md | 61 ++++++++++++++++++----------------------------- package-lock.json | 4 ++-- package.json | 8 +++---- 4 files changed, 30 insertions(+), 44 deletions(-) diff --git a/LICENSE b/LICENSE index 6b9904e..5e4d8b8 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,5 @@ Copyright (c) 2014 Sutoiku, Inc. +Copyright (c) 2021 LeanIX GmbH Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the diff --git a/README.md b/README.md index 66a65de..8659107 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# RedisDOWN [![Build Status](https://travis-ci.org/hmalphettes/redisdown.svg?branch=master)](https://travis-ci.org/hmalphettes/redisdown) +# redis-key-down Redis backend for [LevelUP](https://github.com/rvagg/node-levelup) @@ -6,44 +6,38 @@ Requirements: * redis-2.8 or more recent. * node-12.x -Uses a sorted-set to order the keys and a hash to store the values. +This is a fork of the excellent [redisdown](https://github.com/hmalphettes/redisdown). +Like the original, it uses a sorted-set to order the keys, but the values are stored +as individual keys to make sure that key eviction will not flush all the data when +using Redis as an [LRU cache](https://redis.io/topics/lru-cache). Fetches the ordered key value pairs during iterations with a single redis lua call. -[Abstract-LevelDOWN](https://github.com/rvagg/abstract-leveldown) testsuite is green -except for the ['implicit iterator snapshot'](https://github.com/hmalphettes/redisdown/issues/10). - -# Warning: data migration from redisdown-v0.1.9 -redisdown > v0.1.10 no longer JSON encode/decode itself. Levelup take care of that. -JSON Data written in v0.1.9 is not correctly decoded back to a javascript object in v0.1.10 and above. - -This change was introduced to support binary values in redisdown and escaped the fact that it was breaking backward compatibility for the data. - -Workaround: https://github.com/hmalphettes/redisdown/issues/24#issuecomment-193076281 +[Abstract-LevelDOWN](https://github.com/rvagg/abstract-leveldown) testsuite is green. # Example Copied and pasted from the levelup documentation. -Added the db option when creating the db to use redisdown. +Added the db option when creating the db to use redis-key-down. ``` var levelup = require('levelup') -var redisdown = require('redisdown') +var redisKeyDown = require('redis-key-down') // 1) Create our database, supply location and options. // This will create or open the underlying LevelDB store. -var db = levelup('mydb', { db: redisdown, host: 'localhost', port: 6379 }) +var db = levelup('mydb', { db: redisKeyDown, host: 'localhost', port: 6379 }) // If you use sentinel/cluster mode, you must use a single slot to store the values thanks to a `{hash}` -//var db = levelup('{thehash}path', { db: redisdown }); +//var db = levelup('{thehash}path', { db: redisKeyDown }); // if you already have a redis client -//var db = levelup('mydb', { db: redisdown, redis: redisClient }) +//var db = levelup('mydb', { db: redisKeyDown, redis: redisClient }) // if you use an URL environment variable -//var db = levelup('mydb', { db: redisdown, url: process.env.REDIS_URL }) +//var db = levelup('mydb', { db: redisKeyDown, url: process.env.REDIS_URL }) // if you use Redis Cloud on Heroku -//var db = levelup('mydb', { db: redisdown, url: process.env.REDISCLOUD_URL }) +//var db = levelup('mydb', { db: redisKeyDown, url: process.env.REDISCLOUD_URL }) // 2) put a key & value db.put('name', 'LevelUP', function (err) { @@ -62,15 +56,15 @@ db.put('name', 'LevelUP', function (err) { # API -------------------------------------------------------- -### redisdown(location) -redisdown(location) returns a new **RedisDOWN** instance. `location` is a String pointing at the root namespace of the data in redis. +### redisKeyDown(location) +redisKeyDown(location) returns a new **RedisDOWN** instance. `location` is a String pointing at the root namespace of the data in redis. -* `location+':h'` is the hash where the values are stored. +* `location+'$'+key` is where the values are stored. * `location+':z'` is the set where the keys are sorted. -------------------------------------------------------- - -### redisdown#open([options, ]callback) + +### redisKeyDown#open([options, ]callback) open() is an instance method on an existing database object. options is a hash that is passed to the redis library to create a redis client: @@ -78,17 +72,17 @@ options is a hash that is passed to the redis library to create a redis client: * `highWaterMark` number of values to fetch in one redis call for iteration. Defaults to 256. * `port` redis port. Defaults to '127.0.0.1' * `host` redis host. Defaults to 6379 -* `redis` already configured redis client. redisDown will not open or close it. host and port and all other redis options are ignored. +* `redis` already configured redis client. redisKeyDown will not open or close it. host and port and all other redis options are ignored. * Other options: https://github.com/mranney/node_redis#rediscreateclientport-host-options ----------------------------------- - -### redisdown.destroy(location, [options, ]callback) + +### redisKeyDown.destroy(location, [options, ]callback) destroy(location) is used to completely delete all data in redis related to the location. ----------------------------------- - -### redisdown#batch([{type: 'put', key: 'foo1', value: 'bar1' [, prefix: 'subsection']}, ...]) + +### redisKeyDown#batch([{type: 'put', key: 'foo1', value: 'bar1' [, prefix: 'subsection']}, ...]) batch() supports an exra property `prefix` to store/retrieve/delete a key in a specific namespace of the redis DB. It is useful to support sublevel-ish batch operations: https://github.com/dominictarr/level-sublevel#batches and is well supported by redis. @@ -103,12 +97,3 @@ Passing a levelup instance is demonstrated here: https://github.com/hmalphettes/ The script will install the extra required dependencies. It works for me. - -# LICENSE -redisdown is freely distributable under the term of the MIT License. -Copyright: Sutoiku Inc 2014. - -If you need something different, let me know. - -# HELP Wanted -- Collation: do we need to worry about this? diff --git a/package-lock.json b/package-lock.json index a1dc853..c23145b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { - "name": "redisdown", - "version": "0.1.13", + "name": "redis-key-down", + "version": "0.1.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 93a5a25..3807ecc 100644 --- a/package.json +++ b/package.json @@ -1,10 +1,10 @@ { - "name": "redisdown", - "description": "redis adapter for levelup", - "version": "0.1.13", + "name": "redis-key-down", + "description": "redis adapter for levelup that uses individual keys for values", + "version": "0.1.0", "repository": { "type": "git", - "url": "https://github.com/hmalphettes/redisdown.git" + "url": "https://github.com/leanix/redis-key-down.git" }, "main": "index.js", "scripts": { From 0710fe2943d3863404d89d110c75cd1b91f34933 Mon Sep 17 00:00:00 2001 From: Guido Lorenz Date: Tue, 23 Feb 2021 17:02:41 +0100 Subject: [PATCH 7/9] Don't try to delete obsolete ':h' key --- index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index c9965fb..51f4fac 100644 --- a/index.js +++ b/index.js @@ -232,7 +232,7 @@ RedisDown.destroy = function (location, options, callback) { } var sanitizedLocation = sanitizeLocation(location); var client = redisLib.createClient(options.port, options.host, options); - client.del(sanitizedLocation + ':h', sanitizedLocation + ':z', function (e) { + client.del(sanitizedLocation + ':z', function (e) { client.quit(); callback(e); }); @@ -246,7 +246,7 @@ RedisDown.prototype.destroy = function (doClose, callback) { doClose = true; } var self = this; - this.db.del(this.location + ':h', this.location + ':z', function (e) { + this.db.del(this.location + ':z', function (e) { if (doClose) { self.close(callback); } else { From 485114f37611008b8fb72307e46e832efcb1d309 Mon Sep 17 00:00:00 2001 From: Guido Lorenz Date: Thu, 25 Feb 2021 10:54:49 +0100 Subject: [PATCH 8/9] Serialize all keys and values using abstract-leveldown callbacks `_serializeKey()` and `_serializeValue` are called automatically by abstract-leveldown before parameters are handed over to `_get()`, `_put()` etc. --- index.js | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/index.js b/index.js index 51f4fac..b9a55b3 100644 --- a/index.js +++ b/index.js @@ -110,6 +110,14 @@ RedisDown.prototype._open = function (options, callback) { } }; +RedisDown.prototype._serializeKey = function (key) { + return toBufferOrString(key); +} + +RedisDown.prototype._serializeValue = function (value) { + return toBufferOrString(value); +} + RedisDown.prototype._get = function (key, options, callback) { this.db.get(valueKey(this.location, key), function (e, v) { if (e) { @@ -134,11 +142,11 @@ RedisDown.prototype._get = function (key, options, callback) { }); }; -RedisDown.prototype._put = function (key, rawvalue, opt, callback) { - if (typeof rawvalue === 'undefined' || rawvalue === null) { - rawvalue = ''; +RedisDown.prototype._put = function (key, value, opt, callback) { + if (typeof value === 'undefined' || value === null) { + value = ''; } - this.__exec(this.__appendPutCmd([], key, toBufferOrString(rawvalue)), callback); + this.__exec(this.__appendPutCmd([], key, value), callback); }; RedisDown.prototype._del = function (key, opt, cb) { @@ -166,14 +174,12 @@ RedisDown.prototype.__getPrefix = function (prefix) { RedisDown.prototype.__appendPutCmd = function (commandList, key, value, prefix) { var resolvedPrefix = this.__getPrefix(prefix); - key = toBufferOrString(key); commandList.push(['set', valueKey(resolvedPrefix, key), value === undefined ? '' : value]); commandList.push(['zadd', resolvedPrefix + ':z', 0, key]); return commandList; }; RedisDown.prototype.__appendDelCmd = function (commandList, key, prefix) { - key = toBufferOrString(key); var resolvedPrefix = this.__getPrefix(prefix); commandList.push(['del', valueKey(resolvedPrefix, key)]); commandList.push(['zrem', resolvedPrefix + ':z', key]); From f5e60cecbd364f7b048674bb1d6790ac88ca1d4e Mon Sep 17 00:00:00 2001 From: Guido Lorenz Date: Thu, 25 Feb 2021 10:59:03 +0100 Subject: [PATCH 9/9] Support `Buffer` keys as well --- index.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index b9a55b3..2e7637d 100644 --- a/index.js +++ b/index.js @@ -315,7 +315,12 @@ function toBufferOrString(key) { } function valueKey(location, key) { - return location + '$' + key; + var prefix = location + '$'; + if (Buffer.isBuffer(key)) { + return Buffer.concat([Buffer.from(prefix), key]); + } else { + return prefix + key; + } } RedisDown.reset = function (callback) {