Add spent addresses export test

python-regression/IXI/export-spent/README.md

@@ -0,0 +1,18 @@
+# Spent
+
+Copy or clone source code into your `ixi` directory such that it can be found as `ixi/Spent/{index.js, package.json}`. 
+Your node may be running at this time, and it will hot-load the script. 
+After you've cloned it, and with a running iri node, run the following command to generate the spent addresses file:
+
+```
+curl http://localhost:14265 -X POST -H 'X-IOTA-API-Version: 1.4.1' -H 'Content-Type: application/json'   -d '{"command": "Spent.generateSpentAddressesFile"}'
+```
+
+Generated file will start with the timestamp of generation start.
+Every next line is a human-readable line with the hash of a spent address.
+
+-----
+
+#### Troubleshooting:
+
+

python-regression/IXI/export-spent/index.js

@@ -0,0 +1,79 @@
+var System = java.lang.System;
+
+var spentAddressProvider = IOTA.spentAddressesProvider
+
+var iri = com.iota.iri;
+var Callable = iri.service.CallableRequest;
+var Response = iri.service.dto.IXIResponse;
+var ErrorResponse = iri.service.dto.ErrorResponse;
+
+var fileName = "spentAddresses.txt";
+
+// Log using logger of the ixi class
+var log = org.slf4j.LoggerFactory.getLogger(iri.IXI.class);
+
+/*
+curl http://localhost:14265 -X POST -H 'X-IOTA-API-Version: 1.4.1' -H 'Content-Type: application/json' -d '{"command": "Spent.generateSpentAddressesFile"}'
+*/
+function generate(request) {
+    var file = new java.io.File(fileName);
+    if (file.exists()){
+        if (file.isDirectory()){
+            log.error("Found a directory called {}, aborting!");
+            return ErrorResponse.create("Failed to create spent address due to {} beeing a folder", fileName);
+        } else {
+            log.info("{} already exists.. Overwriting!", fileName);
+        }
+    } else {
+        file.createNewFile();
+    }
+
+    var hashes = spentAddressProvider.getAllAddresses();
+    var writer;
+    var checksum;
+    try {
+        // False for always overwriting
+        writer = new java.io.FileWriter(file, false); 
+
+        var separator = System.getProperty("line.separator");
+
+        // Start with the time
+        writer.write(System.currentTimeMillis() + separator);
+
+        // Create a digest
+        var digest = java.security.MessageDigest.getInstance("SHA-256");
+        for (var i=0; i<hashes.length; i++){
+            // Update with hash bytes
+            digest.update(hashes[i].bytes());
+        }
+
+        // Write checksum
+        checksum = java.lang.String.format("%064x", new java.math.BigInteger(1, digest.digest()));
+        writer.write(checksum + separator);
+
+        // Write amount of addresses
+        writer.write(hashes.length.toString() + separator);
+
+        // Write spent addresses
+        for (var i=0; i<hashes.length; i++){
+            var hash = hashes[i].toString();
+
+            writer.write(hash + separator);
+        }
+
+        writer.close();
+    } catch (error){
+        if (writer){
+            writer.close();
+        }
+    }
+
+    return Response.create({
+        amount: hashes.length,
+        fileName: fileName,
+        sizeInMb: new java.lang.Integer(file.length() / (1024 * 1024)),
+        checksum: checksum
+    });
+}
+
+API.put("generateSpentAddressesFile", new Callable({ call: generate }))

python-regression/IXI/export-spent/package.json

@@ -0,0 +1,3 @@
+{
+  "main": "index.js"
+}

python-regression/IXI/merge-spent/README.md

@@ -0,0 +1,16 @@
+# Merge
+
+Copy or clone source code into your `ixi` directory such that it can be found as `ixi/Merge/{index.js, package.json}`. 
+Your node may be running at this time, and it will hot-load the script. 
+After you've cloned it, and with a running iri node, run the following command to merge your spent-addresses-db with the supplied files:
+
+```
+curl http://localhost:14265 -X POST -H 'X-IOTA-API-Version: 1.4.1' -H 'Content-Type: application/json'   -d '{"command": "Merge.mergeSpentAddresses", "files": ["spentAddresses.txt"]}'
+```
+
+-----
+
+#### Troubleshooting:
+
+Make sure the file you passed to the node exists on the node. If you do not supply the files array, we automatically search for `spentAddresses.txt`
+

python-regression/IXI/merge-spent/index.js

@@ -0,0 +1,147 @@
+var System = java.lang.System;
+
+var spentAddressProvider = IOTA.spentAddressesProvider
+
+var iri = com.iota.iri;
+var Callable = iri.service.CallableRequest;
+var Response = iri.service.dto.IXIResponse;
+var ErrorResponse = iri.service.dto.ErrorResponse;
+
+var defaultSpentFileName = "spentAddresses.txt";
+
+// Log using logger of the ixi class
+var log = org.slf4j.LoggerFactory.getLogger(iri.IXI.class);
+
+function checkChecksumFor(file){
+    var digest = java.security.MessageDigest.getInstance("SHA-256");
+
+    var stream = new java.io.FileInputStream(file);
+    var targetReader = new java.io.BufferedReader(new java.io.InputStreamReader(stream));
+    var expectedChecksum;
+
+    try {
+        var firstLine = true;
+        while (line = targetReader.readLine()){
+            if (firstLine){
+                firstLine = false;
+            } else if (line.length() === 64){
+                // Checksum
+                expectedChecksum = line;
+            } else if (line.length() === 81){
+                // Hash
+                digest.update(line.getBytes());
+            }
+        }
+
+        targetReader.close();
+        stream.close();
+    } catch (error){
+        targetReader.close();
+        stream.close();
+
+        log.error("Caught error during read: {}", error);
+        return false;
+    }
+
+    if (!expectedChecksum){
+        log.error("{} did not have a checksum", file.getName());
+        return false;
+    }
+
+    return expectedChecksum !== checksumFromBytes(digest.digest());
+}
+
+function updateSpentAddressesWith(file){
+    if (!checkChecksumFor(file)){
+        return {
+            "error": "Checksum mismatch for file " + file.getName()
+        }
+    }
+
+    var stream = new java.io.FileInputStream(file);
+    var targetReader = new java.io.BufferedReader(new java.io.InputStreamReader(stream));
+    var amount;
+
+    try {
+        var firstLine = true;
+        var i=0;
+        while (line = targetReader.readLine()){
+            if (firstLine){
+                log.info("Importing spent addresses from {} generated at {}", file.getName(), new java.util.Date(Number(line)));
+                firstLine = false;
+            } else if (line.length() === 81){
+                var hash = com.iota.iri.model.HashFactory.ADDRESS.create(line);
+                try {
+                    spentAddressProvider.saveAddress(hash);
+                    i++;
+                } catch (error){
+                    log.error("Caught error during saveAddress: {}", error);
+                }
+            } else if (line.length() !== 64) {
+                // This must be the amount
+                amount = Number(line);
+                log.info("Importing {} spent addresses", amount);
+            }
+
+            if (i % ((amount / 10)|0) === 0 && i !== 0){
+                log.info("Importing spent addresses {}%..", (i/amount*100)|0);
+            }
+        }
+    } catch (error){
+        targetReader.close();
+        stream.close();
+
+        log.error("Caught error during read: {}", error);
+        return {
+            error: error
+        }
+    }
+
+    return {
+        amount: amount
+    }
+}
+
+function checksumFromBytes(bytes){
+    return java.lang.String.format("%064x", new java.math.BigInteger(1, bytes));
+}
+
+/*
+curl http://localhost:14265 -X POST -H 'X-IOTA-API-Version: 1.4.1' -H 'Content-Type: application/json' -d '{"command": "Merge.mergeSpentAddresses"}'
+*/
+function generate(request) {
+    var files = request["files"];
+    if (!files){
+        files = [defaultSpentFileName];
+    }
+
+    var errors = [];
+    var total = 0;
+    for (var i=0; i<files.length; i++){
+        var fileName = files[i];
+
+        var file = new java.io.File(fileName);
+        if (file.exists()){
+            if (file.isDirectory()){
+                log.error("Found a directory called {}, ignoring!", fileName);
+            } else {
+                var result = updateSpentAddressesWith(file);
+                if (result["error"]){
+                    errors.push(result["error"]);
+                    log.error(result["error"]);
+                } else {
+                    total += result["amount"];
+                }
+            }
+        } else {
+            log.error("File {} did not exist on disk.. ignoring!", fileName)
+        }
+    }
+
+    return Response.create({
+        errors: errors,
+        amountAdded: total
+    });
+}
+
+API.put("mergeSpentAddresses", new Callable({ call: generate }))

python-regression/IXI/merge-spent/package.json

@@ -0,0 +1,3 @@
+{
+  "main": "index.js"
+}

python-regression/IXI/merge-spent/spentAddresses.txt

@@ -0,0 +1,77 @@
+1564941242653
+919432abf85f939057f020d1a589f4d5513ff8890b0fb6b8e2242f90fa9158bc
+74
+E9XGIE9NIUTVUAXT9TKYKBOXIWBT9TEWEVEJIDIE9UFQASMHAKHCVZCMFVUYKAGLBQPFYMTPFKKAKEXHC
+IIZSMXFJU9BTSFWUOFORJSCBKZVFQWCZZIAMDJOMVHOTUSCLFCWMBOXZGGFEWOODTCQFRVGYYBLWOFPCC
+J9HIRH9FRTOSSZR9VPWUAPDAAVHR9OICIBUBGDADYITES99NZGDCBBNJZNBAWFFNJYUIZFZTXMBAAWVPY
+JIPNWEVPMJEWRWLXYBA9O9YGZTSRFYJOBWBIPEKWUGHKZ9SHSGUACRLFRSZOFWGXCXBR9UZLHTEUICSKC
+WSVPTTWNHYGCUELJDLXFABPXFINIGFWBZCXIPVGIGIVWYDDGBDPMJOFBVFPSSDYQIXZVU9ZSYTHDFKXPC
+XAJU9KZCWMGSIYRHAGZEJHHEUINTKNLCHVNKQOVHKWDFKAHFYLFUMEZAPRVJYFUYCQJBMVGJSSJ9GZWKW
+BJEIKNOB9ODWPFPOWGROWDHPWGKPKKVLDMAFNJGIASTFLQYP9XBPPSASTDZLEZNHHMAVBOHPJDBCFBIKC
+BJUSWLBDNVILKVTHLDDYCURCXGRWWKYWLMISUSHYHKAINDIBGIW9DSUYEZAGDEUDDXZRFWEHXAFNEDYPA
+GAPFKCY9YWHJPZORPZJBRHBMDKAFSPYQWDNOOXKCHQTUJKAGXUVVZPCU9UVPEBGXYBNY9KTNAWH9GZYHD
+IJXTTQNRRWGHQQHEFQVOZXUZHULZPPQRHTOGNMUUIQJDKNHSWWQWCJFMAQYLFJPPBSLBIXSHYQAVA9QSB
+AKNAEKBFCPWDTAUTHKXRQZOSRKH9UYKUIZDOSVNCNNPPSJHGDPEAGBWSGZXHNPUE9EKKPXXIKRLJSDJL9
+ATZIRNCVIKHHGRQEQKOZXXGYJYHIZVHCSGODEILFJANBEDMQGW9LVRCPLEGYPLPOJS9EBCKIHEKWRSFGW
+BTADYWDPOUIAFSEEEF9OIPFEICIRDWL9COSWUDWWXJRIJBOE9JBXMGDTXJMVBBTC9ILDTXRNBOKZQGGDY
+GBYOUZHUCNHIAUVC9ZJVROJYLRQ9H9AAOMAQSOZJJRWVODYIDQTSVYIFFTPWI9XIQDUDXZUHRTEVHUQPB
+HBAZYGFHLEJHEDXAJZDX9BPOCJDOZHYEHOFUZZDRCNBYG9DGBXSFGUNSV9ACOJSPOVBUGCIQDLHPGYQDC
+KTQZ9OQCZKEKEVSMSCZOPRKFZKBTRJGZFYRLJGNSDIBYEMOKOZOE9IOVRKRKQDXLVVPSKHZQDPRLNIWXZ
+MTRTRTT9TQROXWRMQKCO9TDNLNEAOYLSURFMQSFMGIMROOYURHEYKEDZ9UCZWTAYLUUTXGHORKEWITSBB
+QUOXBIARUSOAFVXGDEJDAKGIMWHOWQOUXDIW9HOWDGQGJCSJFGBWYND9VLBXPMCCIUFCCUONVTPNXVOMW
+QUUCEEVSSEEMM9ADNWN9BYYDIRKCCFCVZELXKOGNWHDDAOMXRYKMKSZQXX9HWGJXFQXAZOLXCUVSUDZWW
+RLWGTVJMOPCKULQUCIFFXJAAMPJ9ZWMGUGCOHLYHTDERVBPHGWZJQRPJEY9P99CK9JJNGYFRFTHMSCBZX
+RCMHAEEEXMQ9WXLGBDELTJQRSGTSJRJCRRKXCGHKET9QDEMXKTJDSHXHLAZUHVYXDBGQZHWUAVNSHVYTC
+SCRESWQRWCLPKTUZKEXVZHYCCIUAOQMENGXX9ACJTTDILXNGGLIG9ETULZSOGFUFANZJEOASBVJYIQEEY
+VUESCHUUWO9PGOIQCOIZTNIKPQRVAJYDWMSHGIBKCZBJLFAHFXXCPVAYE9WNKLAWBRMNWPXQONIZQOZFX
+VCPQJCEXEHHDCCRXDTDNU9HSDKDWHTDLHFZFLNTEQ9ZDETBKGWFBFODRKGZBMGLUOAVAGSRHNMELOGYMC
+XLQAHRVABKPGZQLBVJXZDDCJGGOHHELNCJERNXBRBYUSPYRUNK9O9TMUATQAZOOLOYHTSKAWRHGWGWYQC
+YCSGZIDYEWIMWNSI9AFJXYNFLBBVPQKHD9QZXLRLUKMJYQKBMRXXRZK9SSMRNXTEHBYQISLPHKJXAVIJY
+ZCPEQRRACUIC9UWNOWQFUNZPZLLTVUTKVPBDSAUDKRKEJFYSO9A9KUMCOASOGVGRXXPXSEXASDWMQMMKD
+GCTWDUKTZBSFJHWLQYIT9ZDME9CETZYECDXXWGRRVJP9Y9ORYYXPIMDSNYHCVUCHXNKL9RRZCSDDOQXPD
+TVEJWJKJXQQAGXAWVTKSPCKCDIJBTI9BCWDNYTQLOYZYWYQ9SBTHOSHMXTCKSJESIKKXWHRUWUPKFFSHB
+YVBQBXL9NGLLYMOIZVCVBLYGVQYBZBQGQZEATZDIXASQZDJORJVI9OHFRTXJBWPJYPYZVQNEBAWCPRGED
+FVWDKIUIIVKHUQGRZK9TNOJYGADWOWOHEMQDMQNRPZIIBKQRVBOB9FCTHRHDFL9HDFHRKCUJRDDDTRK9X
+HVWJNURLQXTZNEPGTNRGRKYKGARZRQTMJJSZRLXWFXNTXU9JEVHCMOUJNRZWYYLPK9BRPSVTC9NLDIHVD
+ONYVETHBKWWAPPPKHATQYFARFDFAYWFUKPJICFREJMSSQBAMQJIBYOUVUUXRSSDHTDKEHR9GAVWSIMPPD
+ONJGXDP9YXUTQCOGRZCDIMFPTAXBQZJCAFLWBQCQWDKXKCEOUNFSTZTL9SZEQDNKLHBPUKWGA9OAAIPQB
+QEQOEGQFJQS9HRB9NQRQWGFVOIIFWQLQGDKABSJLVPQWMDNKJEBALFGALMCHDXULFTJPNUXMBSNF9CTOZ
+SESVJCACFVYECAUBNHTDLUJGE9FPVSTHSOBZHBENBXJRGRCYHZLEWE9CHVCVGBGYLNSUSSCLLNBALIBWZ
+VWXVJNRFXYCTZLFTLVYJMYSMZZ9YGUFDZPXWTPBQUOWCJPM9GXFXVRUFMZVIFVLJSZCVCRJYIMJVYBMXD
+WWPVJFNLWWVAWFKGTQJOODBLWMDVCZGYSNMUXABVAGXPYZHIBEPEZCKDVHJGAMOXDUBCELRMQHPSJF9YZ
+YWF9K9ZSJAKJUOFU9ASSUZZGNIXXAFVAH9PSYXHAIVQFCRTCCIVBYPZWVYGTKD9MVUVYTPYPILUIMPSAC
+YEHIYYBVLFMUJINGOITAHEXSANHSNUFD9BCXWEILDSKQLPSZCLZBPHHRMLZ9WFVWESCDKJTYNBVJTREJX
+9EP9THBVCINPBUAFGDKURBYGNFSESPOO9P9KNHGCTNJ9FHP9GARCBDWWQGNKMPYQMCP9HSHAGYSHGVPM9
+ANXSPRMCCNVLNQCLLQXAOVWFGME9YZKNOOWJVMVGZWSWBNJYDMMPYTEXTZKJVYDNLD9GDEJV9EKMAYRKW
+JNEZEOQTCBTVTHBUBJ9FTQVZCYLVEAZN9DDYWCSWANYDNP9SOKAWECEAQLUPKTWVGWTEGAOYFQOKT9NNY
+MWUZBWXOUK9JPELVGKLDMNVOKWLBJXZZQBIDM9JHT9WPXOANMKPBQQWBAWLZNTCXY9UBSHAWEGHPCYABW
+NOGIBT9PFYXDHHOVUDPCLBEVAPUDCTPKMFCEOBSFSLDWGETLZWZUKEHHIFFJBKSGJZOEDVAVTDHSSHYAW
+UOG9EWIYEEUX99GCPVBYLZDMUN9KYUFMISTRYPVLFYH9YZHRMWLJEAHOMLURTRSPSGP9YSTENDKJIYFOD
+VF9RLBEGCUWVERUDDJDPBNSFYQSMZHBNHIUTRVXSIUXATOQVVUUGNVTVXDKPR9LXX9QPGZPVPXMNLJ9NY
+WXYMWQJXFONABNQY9DAIKDCYLYPJILCTRFMSYYKHSIM9KMAXRPGZPISAEPLYRHMULATBCONAKQVAVKWXY
+ZOJBEPNOJFNCSSUMHPWJUJAHROV9BGMFYIWLPTNDBGHTQGGGAULYEF9URYDMHYIAOAIPSVEFWPPRNSQMB
+GOLZMMKTUKXFOKTVHJAEDEJBJRVIKTNVHTOVXYVMPKQC9UDGSCGORUB9MDGIAEIAMOBPXOQRAKKRTNIBW
+JXKLEVRRBFKNDTPNLBTACWNVNOQEBZIYXLKGKSUHGN9FPUSKBJKUWYTCLCAQAEMBFSR9AUMVPWQKBCEKY
+KFFANCXOTO9FJR9VAYYCNOGYWETJR9Y9DMBMXJAHZOWOTUCCNTQKSKRLZANQXQAGYHPRJEQAKKYLKQCCC
+KONZASMDATPELFFNJBN9FVEFMCDZFPPNAXMWSEHAVIKSXWWZTAG9YXLNB9OP9EQTCWIATRSOTEWWYTHIX
+NPHVXKBQRGRDCXYZHAM9LBAWA9AKGQEJQDSEKWDA9HNQXFKJFXRKRRIZDRWWIFJYYUSUGG9RRT9SVCGA9
+SYQJNUMMGBZYCJHDUJONTBFOTEVJPFRLMONIDCTQIGEPMGZMVZIJGPZ9KER9VXLZ9XWNBKDWGZFOUXKUW
+VP9VAWKCTGCLUVJXBTWGFMSMBQIJEVLYJRLSKCDKIZCOJWGEHUMBZQODCY99K9MAPCWDTSERSGAFEKWHA
+XPLVKUMSJJFMWZYKLRMSVCLXFEQI9PYYTVGKFLHAQJHBELYIFHOQNAPZPFRHTGSOLMRASSFWZHZSJPBRA
+YPLISWKQDPMQETDVMFAASRKCREPQXVKWEXEKWNP9ANGWWXKNVYL9ZNYB9IOVOFFBLGRY9IENRZIMSICOW
+ZYEAOJKJIUDMHVFKYKUUEKKECJTAZERLBDYEJGFSDANJOSRC9IECFSADRCOWCZZFABCYOKCZNNJLPEYOY
+FGSIFYCXSTHLVIJGCRCI9W9EAZSIHHPTUFDSSTVBAGK9RGBFRMZYXTELFXANVNHYN9MWBFXMVGJVVHIBC
+LGVJHGSYHLFRJUGKBISCTESLKBYWGRBGAOKTYPHYBEINZCTIVZXLZYMKTFIAFSESQXXCBELWPEBBVJSCD
+XZUOAUEJVZKYJTBOLVBNMFUZDKMUSJXNX9CEOMTLJMHBZPCOGBJTSMERJYQMPZQKNNWAPCFPEZWJMLYYD
+YZEUZBTTNIQPVXKBLDUDFMYZXNHIGNYDGQTDMIV9QCDPLYJQYFIWOMEUNBQ9FMALSECZBVDPIIMFSCHX9
+9ZSTBPRFDLUPABAWYLFPXBPGFPLYOE9UVXWKZEBJKIPRXLEJWNSFOGLNZGOVTSFAXKQNGNETXOSITHPHC
+AHVFIROLEEC9JIIODOXBLW9EWJNVQJGIFPPRXZUJVUP9EIDVTXIQFQDPWRWRJSIKUIYA9FYRLERBKZTQB
+AZZRWFXTXHXHGDPEXNBFB99WHYFGWYJASSBGXAKVRBDUPKVTYF9BIKIZOKZHFYTWNEQYNMKHZSDYYFRS9
+BZDCYJFZXXHHKMTB9HRNYL9MBZBEZLJREWCCCIIKYPSXQDTUHUDAPTNWWECOTCLKDY9ZTHDATQCMFXQTB
+BQGDQCVEBOEW9HEIJLXXTFHXCWBJJOGQZGTHDSVWXYZOHNJTOQNGRJKIFDWFZXJRNBGPGQTOOZDYQZBTD
+JZRYFAN9FDGCKM9COJJZETMPLLRFNZOWRWSBPKSIYSGJEPZHMXXHEQGGJVYKBNVRVBMBEDWTPRJBTKBSX
+JQUJK9TLSCQYDPOEONDKV9JKENKT9GFJYTGUIMTULWCCLGCM9LAHIFBEKLMDJSLEVMOHZNWCI9JQJJUXX
+MQZCSDKAM9FXWX9XVD9TGATFDERBXLPFJCGLNCOUJXQ9GOBWUZEPDYBBLQGYFGRBFIWFRBCITOCVKWNH9
+Q9NHVVWYGINGUFEOPEWGOWNOXSYURZNTNTLAGBRHKMUCZMYULNVMLGYIPIKLHOUUAUWVJPLQZLFQOSCHC
+SRYZZ9C9ZCZZVYLOCTXNDJKDPJTSDQQHNLMIFYUWHMXUOATOKKUJNUKLAKZAPFPFMSVOGOUEACYUSODWW
+YI9DPIGBYPFAND9UKONCHTIDUJWOXSNTBDX9KXZIPNF99HPFIUMNCSKHCGOALTECQSGNKPJZNHWYQUXRY

python-regression/setup.py

@@ -9,6 +9,7 @@
         'pyota',
         'aloe',
         'pyyaml',
+        'kubernetes',
         ]
       )