diff --git a/lib/rfs.js b/lib/rfs.js index e61d2f58c..ef54377e3 100644 --- a/lib/rfs.js +++ b/lib/rfs.js @@ -3,12 +3,32 @@ function Store(opt){ opt.log = opt.log || console.log; opt.file = String(opt.file || 'radata'); var fs = require('fs'), u; - + var sha256 = require('./sha256'); var store = function Store(){}; if(Store[opt.file]){ console.log("Warning: reusing same fs store and options as 1st."); return Store[opt.file]; } + + if(!fs.existsSync(opt.file)){ fs.mkdirSync(opt.file) } + + const indexFile = opt.file+'/'+opt.file+'.idx' + var logger = fs.createWriteStream(indexFile, { + flags: 'a' // 'a' means appending (old data will be preserved) + }) + let cleanupCalled = false + const cleanupIndex = (err) => { + if(cleanupCalled) return; + cleanupCalled = true; + logger.end(() => { + process.exit() + }) + } + [`exit`, `SIGINT`, `SIGUSR1`, `SIGUSR2`, `uncaughtException`, `SIGTERM`].forEach((eventType) => { + process.on(eventType, cleanupIndex); + }) + var index = {} + Store[opt.file] = store; var puts = {}; @@ -16,16 +36,18 @@ function Store(opt){ store.put = function(file, data, cb){ puts[file] = data; var random = Math.random().toString(36).slice(-3); - var tmp = opt.file+'-'+file+'-'+random+'.tmp'; + const hash = sha256.hash(file); + var tmp = opt.file+'-'+hash+'-'+random+'.tmp'; fs.writeFile(tmp, data, function(err, ok){ delete puts[file]; if(err){ return cb(err) } - move(tmp, opt.file+'/'+file, cb); + move(tmp, file,hash, cb); }); }; store.get = function(file, cb){ var tmp; // this took 3s+? if(tmp = puts[file]){ cb(u, tmp); return } - fs.readFile(opt.file+'/'+file, function(err, data){ + const hash = sha256.hash(file); + fs.readFile(opt.file+'/'+hash, function(err, data){ if(err){ if('ENOENT' === (err.code||'').toUpperCase()){ return cb(); @@ -36,9 +58,18 @@ function Store(opt){ }); }; - if(!fs.existsSync(opt.file)){ fs.mkdirSync(opt.file) } - function move(oldPath, newPath, cb) { + function move(oldPath, file, hash, cb) { + const newPath = opt.file+'/'+hash; + if(!index[hash]) + { + index[hash] = true; + + logger.write(file+'\n',function(err){ + if(err) + cb(err) + }) + } fs.rename(oldPath, newPath, function (err) { if (err) { if (err.code === 'EXDEV') { @@ -63,10 +94,27 @@ function Store(opt){ }; store.list = function(cb, match, params, cbs){ - var dir = fs.readdirSync(opt.file); - dir.forEach(function(file){ - cb(file); - }) + if(fs.existsSync(indexFile)) + { + fs.readFileSync(indexFile,'utf8').split(/\r?\n/).forEach(fn => { + //skip the empty line + if(!fn) return; + index[sha256.hash(fn)] = true + cb(fn) + }); + } + //try to perform upgrade if index is empty + if(Object.keys(index).length === 0) { + const re = /^[0-9A-Fa-f]+$/g; + const dir = fs.readdirSync(opt.file); + dir.forEach(function(fn){ + //convert only non 32byte hex filenames (ie not in new format) + if(!fn || fn === opt.file+'.idx' || (fn.length===64 && re.test(fn)) ) return; + const hash = sha256.hash(fn); + move(opt.file+"/"+fn,fn,hash,() => {}) + cb(fn) + }) + } cb(); }; diff --git a/lib/sha256.js b/lib/sha256.js new file mode 100644 index 000000000..290daf32e --- /dev/null +++ b/lib/sha256.js @@ -0,0 +1,180 @@ +/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ +/* SHA-256 (FIPS 180-4) implementation in JavaScript (c) Chris Veness 2002-2019 */ +/* MIT Licence */ +/* www.movable-type.co.uk/scripts/sha256.html */ +/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ + + +/** + * SHA-256 hash function reference implementation. + * + * This is an annotated direct implementation of FIPS 180-4, without any optimisations. It is + * intended to aid understanding of the algorithm rather than for production use. + * + * While it could be used where performance is not critical, I would recommend using the ‘Web + * Cryptography API’ (developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest) for the browser, + * or the ‘crypto’ library (nodejs.org/api/crypto.html#crypto_class_hash) in Node.js. + * + * See csrc.nist.gov/groups/ST/toolkit/secure_hashing.html + * csrc.nist.gov/groups/ST/toolkit/examples.html + */ +class Sha256 { + + /** + * Generates SHA-256 hash of string. + * + * @param {string} msg - (Unicode) string to be hashed. + * @param {Object} [options] + * @param {string} [options.msgFormat=string] - Message format: 'string' for JavaScript string + * (gets converted to UTF-8 for hashing); 'hex-bytes' for string of hex bytes ('616263' ≡ 'abc') . + * @param {string} [options.outFormat=hex] - Output format: 'hex' for string of contiguous + * hex bytes; 'hex-w' for grouping hex bytes into groups of (4 byte / 8 character) words. + * @returns {string} Hash of msg as hex character string. + * + * @example + * import Sha256 from './sha256.js'; + * const hash = Sha256.hash('abc'); // 'ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad' + */ + static hash(msg, options) { + const defaults = { msgFormat: 'string', outFormat: 'hex' }; + const opt = Object.assign(defaults, options); + + // note use throughout this routine of 'n >>> 0' to coerce Number 'n' to unsigned 32-bit integer + + switch (opt.msgFormat) { + default: // default is to convert string to UTF-8, as SHA only deals with byte-streams + case 'string': msg = utf8Encode(msg); break; + case 'hex-bytes':msg = hexBytesToString(msg); break; // mostly for running tests + } + + // constants [§4.2.2] + const K = [ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 ]; + + // initial hash value [§5.3.3] + const H = [ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19 ]; + + // PREPROCESSING [§6.2.1] + + msg += String.fromCharCode(0x80); // add trailing '1' bit (+ 0's padding) to string [§5.1.1] + + // convert string msg into 512-bit blocks (array of 16 32-bit integers) [§5.2.1] + const l = msg.length/4 + 2; // length (in 32-bit integers) of msg + ‘1’ + appended length + const N = Math.ceil(l/16); // number of 16-integer (512-bit) blocks required to hold 'l' ints + const M = new Array(N); // message M is N×16 array of 32-bit integers + + for (let i=0; i>> 32, but since JS converts + // bitwise-op args to 32 bits, we need to simulate this by arithmetic operators + const lenHi = ((msg.length-1)*8) / Math.pow(2, 32); + const lenLo = ((msg.length-1)*8) >>> 0; + M[N-1][14] = Math.floor(lenHi); + M[N-1][15] = lenLo; + + + // HASH COMPUTATION [§6.2.2] + + for (let i=0; i>> 0; + } + + // 2 - initialise working variables a, b, c, d, e, f, g, h with previous hash value + let a = H[0], b = H[1], c = H[2], d = H[3], e = H[4], f = H[5], g = H[6], h = H[7]; + + // 3 - main loop (note '>>> 0' for 'addition modulo 2^32') + for (let t=0; t<64; t++) { + const T1 = h + Sha256.Σ1(e) + Sha256.Ch(e, f, g) + K[t] + W[t]; + const T2 = Sha256.Σ0(a) + Sha256.Maj(a, b, c); + h = g; + g = f; + f = e; + e = (d + T1) >>> 0; + d = c; + c = b; + b = a; + a = (T1 + T2) >>> 0; + } + + // 4 - compute the new intermediate hash value (note '>>> 0' for 'addition modulo 2^32') + H[0] = (H[0]+a) >>> 0; + H[1] = (H[1]+b) >>> 0; + H[2] = (H[2]+c) >>> 0; + H[3] = (H[3]+d) >>> 0; + H[4] = (H[4]+e) >>> 0; + H[5] = (H[5]+f) >>> 0; + H[6] = (H[6]+g) >>> 0; + H[7] = (H[7]+h) >>> 0; + } + + // convert H0..H7 to hex strings (with leading zeros) + for (let h=0; h prev + String.fromCharCode(curr), ''); + } catch (e) { // no TextEncoder available? + return unescape(encodeURIComponent(str)); // monsur.hossa.in/2012/07/20/utf-8-in-javascript.html + } + } + + function hexBytesToString(hexStr) { // convert string of hex numbers to a string of chars (eg '616263' -> 'abc'). + const str = hexStr.replace(' ', ''); // allow space-separated groups + return str=='' ? '' : str.match(/.{2}/g).map(byte => String.fromCharCode(parseInt(byte, 16))).join(''); + } + } + + + + /** + * Rotates right (circular right shift) value x by n positions [§3.2.4]. + * @private + */ + static ROTR(n, x) { + return (x >>> n) | (x << (32-n)); + } + + + /** + * Logical functions [§4.1.2]. + * @private + */ + static Σ0(x) { return Sha256.ROTR(2, x) ^ Sha256.ROTR(13, x) ^ Sha256.ROTR(22, x); } + static Σ1(x) { return Sha256.ROTR(6, x) ^ Sha256.ROTR(11, x) ^ Sha256.ROTR(25, x); } + static σ0(x) { return Sha256.ROTR(7, x) ^ Sha256.ROTR(18, x) ^ (x>>>3); } + static σ1(x) { return Sha256.ROTR(17, x) ^ Sha256.ROTR(19, x) ^ (x>>>10); } + static Ch(x, y, z) { return (x & y) ^ (~x & z); } // 'choice' + static Maj(x, y, z) { return (x & y) ^ (x & z) ^ (y & z); } // 'majority' + +} + + +/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ + +module.exports = Sha256; \ No newline at end of file diff --git a/test/rad/rad.js b/test/rad/rad.js index 28c91dba3..bbf568958 100644 --- a/test/rad/rad.js +++ b/test/rad/rad.js @@ -278,6 +278,18 @@ var names = ["Adalard","Adora","Aia","Albertina","Alfie","Allyn","Amabil","Ammam }); + describe("RAD file hash",function(){ + it("should have index file",function(done){ + let found = false + opt.store.list((fn) => { + if(fn !== undefined) { found = true; return} + else { + expect(found).to.be.ok() + done() + } + }) + }) + }) var ntmp = names; describe('RAD + GUN', function(){ var ochunk = 1000;