merkletree-master/0000755000175000017500000000000013152305166013134 5ustar freefreemerkletree-master/.gitignore0000644000175000017500000000003613152305166015123 0ustar freefreefuzz *.swp *.out *.html *.zip merkletree-master/.travis.yml0000644000175000017500000000051413152305166015245 0ustar freefreelanguage: go go: - 1.7 install: - go get -u github.com/golang/lint/golint - go get -u github.com/kisielk/errcheck - go get -u golang.org/x/tools/cmd/cover - test -z "$(go fmt)" - test -z "$(golint)" script: errcheck && go vet && go test -v -tags=debug -bench=. && go test -covermode=atomic && go install sudo: false merkletree-master/LICENSE0000644000175000017500000000206413152305166014143 0ustar freefreeThe MIT License (MIT) Copyright (c) 2015 Nebulous Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. merkletree-master/Makefile0000644000175000017500000000122713152305166014576 0ustar freefreeall: install REBUILD: @touch debug*.go dependencies: go get -u github.com/dvyukov/go-fuzz/go-fuzz go get -u github.com/dvyukov/go-fuzz/go-fuzz-build install: REBUILD go install test: REBUILD go test -v -tags='debug' -timeout=600s test-short: REBUILD go test -short -v -tags='debug' -timeout=6s cover: REBUILD go test -v -tags='debug' -cover -coverprofile=cover.out go tool cover -html=cover.out -o=cover.html rm cover.out fuzz: REBUILD go install -tags='debug gofuzz' go-fuzz-build github.com/NebulousLabs/merkletree go-fuzz -bin=./merkletree-fuzz.zip -workdir=fuzz .PHONY: all REBUILD dependencies install test test-short cover fuzz benchmark merkletree-master/README.md0000644000175000017500000001334413152305166014420 0ustar freefreemerkletree ---------- merkletree is a Go package for working with [Merkle trees](http://en.wikipedia.org/wiki/Merkle_tree). Specifically, this package is designed to facilitate the generation and verification of "Merkle proofs" — cryptographic proofs that a given subset of data "belongs" to a larger set. BitTorrent, for example, requires downloading many small pieces of a file from many untrusted peers; Merkle proofs allow the downloader to verify that each piece is part of the full file. When sha256 is used as the hashing algorithm, the implementation matches the merkle tree described in RFC 6962, 'Certificate Transparency'. Usage ----- ```go package main import ( "crypto/sha256" "log" "os" "github.com/NebulousLabs/merkletree" ) // All error checking is ignored in the following examples. func main() { // Example 1: Get the merkle root of a file. segmentSize := 4096 // bytes per leaf file, _ := os.Open("myfile") merkleRoot, _ := merkletree.ReaderRoot(file, sha256.New(), segmentSize) // Example 2: Build and verify a proof that the element at segment 7 is in // the merkle root. file.Seek(0, 0) // Offset needs to be set back to 0. proofIndex := uint64(7) merkleRoot, proof, numLeaves, _ := merkletree.BuildReaderProof(file, sha256.New(), segmentSize, proofIndex) verified := VerifyProof(sha256.New(), merkleRoot, proof, proofIndex, numLeaves) // Example 3: Using a Tree to build a merkle tree and get a proof for a // specific index for non-file objects. tree := merkletree.New(sha256.New()) tree.SetIndex(1) tree.Push([]byte("an object - the tree will hash the data after it is pushed")) tree.Push([]byte("another object")) // The merkle root could be obtained by calling tree.Root(), but will also // be provided by tree.Prove() merkleRoot, proof, proofIndex, numLeaves := tree.Prove() //////////////////////////////////////////////// /// Remaining examples deal with cached trees // //////////////////////////////////////////////// // Example 4: Creating a cached set of Merkle roots and then using them in // a cached tree. The cached tree is height 1, meaning that all elements of // the cached tree will be Merkle roots of data with 2 leaves. cachedTree := merkletree.NewCachedTree(sha256.New(), 1) subtree1 := merkletree.New(sha256.New()) subtree1.Push([]byte("first leaf, first subtree")) subtree1.Push([]byte("second leaf, first subtree")) subtree2 := merkletree.New(sha256.New()) subtree2.Push([]byte("first leaf, second subtree")) subtree2.Push([]byte("second leaf, second subtree")) // Using the cached tree, build the merkle root of the 4 leaves. cachedTree.Push(subtree1.Root()) cachedTree.Push(subtree2.Root()) collectiveRoot := cachedTree.Root() // Example 5: Modify the data pushed into subtree 2 and create the Merkle // root, without needing to rehash the data in any other subtree. revisedSubtree2 := merkletree.New(sha256.New()) revisedSubtree2.Push([]byte("first leaf, second subtree")) revisedSubtree2.Push([]byte("second leaf, second subtree, revised")) // Using the cached tree, build the merkle root of the 4 leaves - without // needing to rehash any of the data in subtree1. cachedTree = merkletree.NewCachedTree(sha256.New(), 1) cachedTree.Push(subtree1.Root()) cachedTree.Push(revisedSubtree2.Root()) revisedRoot := cachedTree.Root() // Exapmle 6: Create a proof that leaf 3 (index 2) of the revised root, // found in revisedSubtree2 (at index 0 of the revised subtree), is a part of // the cached set. This is a two stage process - first we must get a proof // that the leaf is a part of revisedSubtree2, and then we must get provide // that proof as input to the cached tree prover. cachedTree = merkletree.NewCachedTree(sha256.New(), 1) cachedTree.SetIndex(2) // leaf at index 2, or the third element which gets inserted. revisedSubtree2 = merkletree.New(sha256.New()) revisedSubtree2.SetIndex(0) revisedSubtree2.Push([]byte("first leaf, second subtree")) revisedSubtree2.Push([]byte("second leaf, second subtree, revised")) _, subtreeProof, _, _ := revisedSubtree2.Prove() // Now we can create the full proof for the cached tree, without having to // rehash any of the elements from subtree1. _, fullProof, _, _ := cachedTree.Prove(subtreeProof) } ``` For more extensive documentation, refer to the [godoc](http://godoc.org/github.com/NebulousLabs/merkletree). Notes ----- This implementation does not retain the entire Merkle tree in memory. Rather, as each new leaf is added to the tree, is it pushed onto a stack as a "subtree of depth 1." If the next element on the stack also has depth 1, the two are combined into a "subtree of depth 2." This process continues until no adjacent elements on the stack have the same depth. (For a nice visual representation of this, play a round of [2048](http://gabrielecirulli.github.io/2048).) This gives a space complexity of O(log(n)), making this implementation suitable for generating Merkle proofs on very large files. (It is not as suitable for generating "batches" of many Merkle proofs on the same file.) Different Merkle tree implementations handle "orphan" leaves in different ways. Our trees conform to the diagrams below; orphan leaves are not duplicated or hashed multiple times. ``` ┌───┴──┐ ┌────┴───┐ ┌─────┴─────┐ ┌──┴──┐ │ ┌──┴──┐ │ ┌──┴──┐ ┌──┴──┐ ┌─┴─┐ ┌─┴─┐ │ ┌─┴─┐ ┌─┴─┐ ┌─┴─┐ ┌─┴─┐ ┌─┴─┐ ┌─┴─┐ │ (5-leaf) (6-leaf) (7-leaf) ``` When using the Reader functions (ReaderRoot and BuildReaderProof), the last segment will not be padded if there are not 'segmentSize' bytes remaining. merkletree-master/cachedtree.go0000644000175000017500000000514713152305166015561 0ustar freefreepackage merkletree import ( "errors" "hash" ) // A CachedTree can be used to build Merkle roots and proofs from the cached // Merkle roots of smaller blocks of data. Each CachedTree has a height, // meaning every element added to the CachedTree is the root of a full Merkle // tree containing 2^height leaves. type CachedTree struct { cachedNodeHeight uint64 trueProofIndex uint64 Tree } // NewCachedTree initializes a CachedTree with a hash object, which will be // used when hashing the input. func NewCachedTree(h hash.Hash, cachedNodeHeight uint64) *CachedTree { return &CachedTree{ cachedNodeHeight: cachedNodeHeight, Tree: Tree{ hash: h, cachedTree: true, }, } } // Prove will create a proof that the leaf at the indicated index is a part of // the data represented by the Merkle root of the Cached Tree. The CachedTree // needs the proof set proving that the index is an element of the cached // element in order to create a correct proof. After proof is called, the // CachedTree is unchanged, and can receive more elements. func (ct *CachedTree) Prove(cachedProofSet [][]byte) (merkleRoot []byte, proofSet [][]byte, proofIndex uint64, numLeaves uint64) { // Determine the proof index within the full tree, and the number of leaves // within the full tree. leavesPerCachedNode := uint64(1) << ct.cachedNodeHeight numLeaves = leavesPerCachedNode * ct.currentIndex // Get the proof set tail, which is generated based entirely on cached // nodes. merkleRoot, proofSetTail, _, _ := ct.Tree.Prove() if len(proofSetTail) < 1 { // The proof was invalid, return 'nil' for the proof set but accurate // values for everything else. return merkleRoot, nil, ct.trueProofIndex, numLeaves } // The full proof set is going to be the input cachedProofSet combined with // the tail proof set. The one caveat is that the tail proof set has an // extra piece of data at the first element - the verifier will assume that // this data exists and therefore it needs to be omitted from the proof // set. proofSet = append(cachedProofSet, proofSetTail[1:]...) return merkleRoot, proofSet, ct.trueProofIndex, numLeaves } // SetIndex will inform the CachedTree of the index of the leaf for which a // storage proof is being created. The index should be the index of the actual // leaf, and not the index of the cached element containing the leaf. SetIndex // must be called on empty CachedTree. func (ct *CachedTree) SetIndex(i uint64) error { if ct.head != nil { return errors.New("cannot call SetIndex on Tree if Tree has not been reset") } ct.trueProofIndex = i return ct.Tree.SetIndex(i / (1 << ct.cachedNodeHeight)) } merkletree-master/cachedtree_test.go0000644000175000017500000002475313152305166016624 0ustar freefreepackage merkletree import ( "bytes" "crypto/sha256" "testing" ) // addSubTree will create a subtree of the desired height using the dataSeed to // seed the data. addSubTree will add the data created in the subtree to the // Tree as well. The tree must have the proveIndex set separately. func addSubTree(height uint64, dataSeed []byte, subtreeProveIndex uint64, fullTree *Tree) (subTree *Tree) { data := sum(sha256.New(), dataSeed) leaves := 1 << height subTree = New(sha256.New()) err := subTree.SetIndex(subtreeProveIndex) if err != nil { panic(err) } for i := 0; i < leaves; i++ { subTree.Push(data) fullTree.Push(data) data = sum(sha256.New(), data) } return subTree } // TestCachedTreeConstruction checks that a CachedTree will correctly build to // the same merkle root as the Tree when using caches at various heights and // lengths. func TestCachedTreeConstruction(t *testing.T) { arbData := [][]byte{ []byte{1}, []byte{2}, []byte{3}, []byte{4}, []byte{5}, []byte{6}, []byte{7}, []byte{8}, } // Test that a CachedTree with no elements will return the same value as a // tree with no elements. tree := New(sha256.New()) cachedTree := NewCachedTree(sha256.New(), 0) if bytes.Compare(tree.Root(), cachedTree.Root()) != 0 { t.Error("empty Tree and empty CachedTree do not match") } // Try comparing the root of a cached tree with one element, where the // cache height is 0. tree = New(sha256.New()) cachedTree = NewCachedTree(sha256.New(), 0) tree.Push(arbData[0]) subRoot := tree.Root() cachedTree.Push(subRoot) if bytes.Compare(tree.Root(), cachedTree.Root()) != 0 { t.Error("naive 1-height Tree and CachedTree do not match") } // Try comparing the root of a cached tree where the cache height is 0, and // there are 3 cached elements. tree = New(sha256.New()) subTree1 := New(sha256.New()) subTree2 := New(sha256.New()) cachedTree = NewCachedTree(sha256.New(), 0) // Create 3 subtrees, one for caching each element. subTree3 := New(sha256.New()) subTree1.Push(arbData[0]) subTree2.Push(arbData[1]) subTree3.Push(arbData[2]) // Pushed the cached roots into the cachedTree. cachedTree.Push(subTree1.Root()) cachedTree.Push(subTree2.Root()) cachedTree.Push(subTree3.Root()) // Create a tree from the original elements. tree.Push(arbData[0]) tree.Push(arbData[1]) tree.Push(arbData[2]) if bytes.Compare(tree.Root(), cachedTree.Root()) != 0 { t.Error("adding 3 len cacheing is causing problems") } // Try comparing the root of a cached tree where the cache height is 1, and // there is 1 cached element. tree = New(sha256.New()) subTree1 = New(sha256.New()) cachedTree = NewCachedTree(sha256.New(), 1) // Build the subtrees to get the cached roots. subTree1.Push(arbData[0]) subTree1.Push(arbData[1]) // Supply the cached roots to the cached tree. cachedTree.Push(subTree1.Root()) // Compare against a formally built tree. tree.Push(arbData[0]) tree.Push(arbData[1]) if bytes.Compare(cachedTree.Root(), tree.Root()) != 0 { t.Error("comparison has failed") } // Mirror the above test, but attempt a mutation, which should cause a // failure. tree = New(sha256.New()) subTree1 = New(sha256.New()) cachedTree = NewCachedTree(sha256.New(), 1) // Build the subtrees to get the cached roots. subTree1.Push(arbData[0]) subTree1.Push(arbData[1]) // Supply the cached roots to the cached tree. cachedTree.Push(subTree1.Root()) // Compare against a formally built tree. tree.Push(arbData[1]) // Intentional mistake. tree.Push(arbData[1]) if bytes.Compare(cachedTree.Root(), tree.Root()) == 0 { t.Error("comparison has succeeded despite mutation") } // Try comparing the root of a cached tree where the cache height is 2, and // there are 5 cached elements. tree = New(sha256.New()) subTree1 = New(sha256.New()) subTree2 = New(sha256.New()) cachedTree = NewCachedTree(sha256.New(), 2) // Build the subtrees to get the cached roots. subTree1.Push(arbData[0]) subTree1.Push(arbData[1]) subTree1.Push(arbData[2]) subTree1.Push(arbData[3]) subTree2.Push(arbData[4]) subTree2.Push(arbData[5]) subTree2.Push(arbData[6]) subTree2.Push(arbData[7]) // Supply the cached roots to the cached tree. cachedTree.Push(subTree1.Root()) cachedTree.Push(subTree1.Root()) cachedTree.Push(subTree1.Root()) cachedTree.Push(subTree1.Root()) cachedTree.Push(subTree2.Root()) // Compare against a formally built tree. for i := 0; i < 4; i++ { for j := 0; j < 4; j++ { tree.Push(arbData[j]) } } for i := 4; i < 8; i++ { tree.Push(arbData[i]) } if bytes.Compare(cachedTree.Root(), tree.Root()) != 0 { t.Error("comparison has failed") } // Try proving on an uninitialized cached tree. cachedTree = NewCachedTree(sha256.New(), 0) _, proofSet, _, _ := cachedTree.Prove(nil) if proofSet != nil { t.Error("proving an empty set resulted in a valid proof?") } cachedTree = NewCachedTree(sha256.New(), 1) _, proofSet, _, _ = cachedTree.Prove(nil) if proofSet != nil { t.Error("proving an empty set resulted in a valid proof?") } cachedTree = NewCachedTree(sha256.New(), 2) _, proofSet, _, _ = cachedTree.Prove(nil) if proofSet != nil { t.Error("proving an empty set resulted in a valid proof?") } // Try creating a cached proof with cache height 1, 2 cached nodes, index // 1. tree = New(sha256.New()) subTree1 = New(sha256.New()) err := subTree1.SetIndex(1) // subtree index 0-1, corresponding to index 1. if err != nil { t.Fatal(err) } subTree2 = New(sha256.New()) cachedTree = NewCachedTree(sha256.New(), 1) err = cachedTree.SetIndex(1) if err != nil { t.Fatal(err) } // Build the subtrees. subTree1.Push(arbData[0]) subTree1.Push(arbData[1]) subTree2.Push(arbData[2]) subTree2.Push(arbData[3]) // Supply the cached root to the cached tree. cachedTree.Push(subTree1.Root()) cachedTree.Push(subTree2.Root()) // Get the root from the tree, to have certainty about integrity. tree.Push(arbData[0]) tree.Push(arbData[1]) tree.Push(arbData[2]) tree.Push(arbData[3]) root := tree.Root() // Construct the proofs. _, subTreeProofSet, _, _ := subTree1.Prove() _, proofSet, proofIndex, numLeaves := cachedTree.Prove(subTreeProofSet) if !VerifyProof(sha256.New(), root, proofSet, proofIndex, numLeaves) { t.Error("proof was unsuccessful") } // Try creating a cached proof with cache height 0, 3 cached nodes, index // 2. tree = New(sha256.New()) subTree1 = New(sha256.New()) subTree2 = New(sha256.New()) subTree3 = New(sha256.New()) err = subTree3.SetIndex(0) // subtree index 2-0, corresponding to index 2. if err != nil { t.Fatal(err) } cachedTree = NewCachedTree(sha256.New(), 0) err = cachedTree.SetIndex(2) if err != nil { t.Fatal(err) } // Build the subtrees. subTree1.Push(arbData[0]) subTree2.Push(arbData[1]) subTree3.Push(arbData[2]) // Supply the cached root to the cached tree. cachedTree.Push(subTree1.Root()) cachedTree.Push(subTree2.Root()) cachedTree.Push(subTree3.Root()) // Get the root from the tree, to have certainty about integrity. tree.Push(arbData[0]) tree.Push(arbData[1]) tree.Push(arbData[2]) root = tree.Root() // Construct the proofs. _, subTreeProofSet, _, _ = subTree3.Prove() _, proofSet, proofIndex, numLeaves = cachedTree.Prove(subTreeProofSet) if !VerifyProof(sha256.New(), root, proofSet, proofIndex, numLeaves) { t.Error("proof was unsuccessful") } // Try creating a cached proof with cache height 2, 3 cached nodes, index // 6. tree = New(sha256.New()) subTree1 = New(sha256.New()) subTree2 = New(sha256.New()) err = subTree2.SetIndex(2) // subtree index 1-2, corresponding to index 6. if err != nil { t.Fatal(err) } subTree3 = New(sha256.New()) cachedTree = NewCachedTree(sha256.New(), 2) err = cachedTree.SetIndex(6) if err != nil { t.Fatal(err) } // Build the subtrees. subTree1.Push(arbData[0]) subTree1.Push(arbData[1]) subTree1.Push(arbData[2]) subTree1.Push(arbData[3]) subTree2.Push(arbData[4]) subTree2.Push(arbData[5]) subTree2.Push(arbData[6]) subTree2.Push(arbData[7]) subTree3.Push(arbData[1]) subTree3.Push(arbData[3]) subTree3.Push(arbData[5]) subTree3.Push(arbData[7]) // Supply the cached root to the cached tree. cachedTree.Push(subTree1.Root()) cachedTree.Push(subTree2.Root()) cachedTree.Push(subTree3.Root()) // Get the root from the tree, to have certainty about integrity. tree.Push(arbData[0]) tree.Push(arbData[1]) tree.Push(arbData[2]) tree.Push(arbData[3]) tree.Push(arbData[4]) tree.Push(arbData[5]) tree.Push(arbData[6]) tree.Push(arbData[7]) tree.Push(arbData[1]) tree.Push(arbData[3]) tree.Push(arbData[5]) tree.Push(arbData[7]) root = tree.Root() // Construct the proofs. _, subTreeProofSet, _, _ = subTree2.Prove() _, proofSet, proofIndex, numLeaves = cachedTree.Prove(subTreeProofSet) if !VerifyProof(sha256.New(), root, proofSet, proofIndex, numLeaves) { t.Error("proof was unsuccessful") } } // TestCachedTreeConstructionAuto uses automation to build out a wide set of // trees of different types to make sure the Cached Tree maintains consistency // with the actual tree. func TestCachedTreeConstructionAuto(t *testing.T) { if testing.Short() { t.SkipNow() } // Build out cached trees with up to 33 cached elements, each height 'h'. for h := uint64(0); h < 5; h++ { n := uint64(1) << h for i := uint64(0); i < 35; i++ { // Try creating a proof at each index. for j := uint64(0); j < i*n; j++ { tree := New(sha256.New()) err := tree.SetIndex(j) if err != nil { t.Fatal(err) } cachedTree := NewCachedTree(sha256.New(), h) err = cachedTree.SetIndex(j) if err != nil { t.Fatal(err) } var subProof [][]byte // Build out 'i' subtrees that form the components of the cached // tree. for k := uint64(0); k < i; k++ { subtree := addSubTree(uint64(h), []byte{byte(k)}, j%n, tree) cachedTree.Push(subtree.Root()) if bytes.Compare(tree.Root(), cachedTree.Root()) != 0 { t.Error("naive 1-height Tree and Cached tree roots do not match") } // Get the proof of the subtree if k == j/n { _, subProof, _, _ = subtree.Prove() } } // Verify that the tree was built correctly. treeRoot, treeProof, treeProofIndex, treeLeaves := tree.Prove() if !VerifyProof(sha256.New(), treeRoot, treeProof, treeProofIndex, treeLeaves) { t.Error("tree problems", i, j) } // Verify that the cached tree was built correctly. cachedRoot, cachedProof, cachedProofIndex, cachedLeaves := cachedTree.Prove(subProof) if !VerifyProof(sha256.New(), cachedRoot, cachedProof, cachedProofIndex, cachedLeaves) { t.Error("cached tree problems", i, j) } } } } } merkletree-master/debugoff.go0000644000175000017500000000044313152305166015245 0ustar freefree// +build !debug package merkletree const ( // DEBUG indicates whether debugging is enabled. When debugging is enabled, // checks are performed on all stateful objects to make sure no supposedly // impossible conditions have occurred. The DEBUG flag is for developers. DEBUG = false ) merkletree-master/debugon.go0000644000175000017500000000044113152305166015105 0ustar freefree// +build debug package merkletree const ( // DEBUG indicates whether debugging is enabled. When debugging is enabled, // checks are performed on all stateful objects to make sure no supposedly // impossible conditions have occurred. The DEBUG flag is for developers. DEBUG = true ) merkletree-master/fuzz.go0000644000175000017500000000166513152305166014471 0ustar freefree// +build gofuzz package merkletree import ( "bytes" "crypto/sha256" ) // Fuzz is called by go-fuzz to look for inputs to BuildReaderProof that will // not verify correctly. func Fuzz(data []byte) int { // Use the first two bytes to determine the proof index. if len(data) < 2 { return -1 } index := 256*uint64(data[0]) + uint64(data[1]) data = data[2:] // Build a reader proof for index 'index' using the remaining data as input // to the reader. '64' is chosen as the only input size because that is the // size relevant to the Sia project. merkleRoot, proofSet, numLeaves, err := BuildReaderProof(bytes.NewReader(data), sha256.New(), 64, index) if err != nil { return 0 } if !VerifyProof(sha256.New(), merkleRoot, proofSet, index, numLeaves) { panic("verification failed!") } // Output is more interesting when there is enough data to contain the // index. if uint64(len(data)) > 64*index { return 1 } return 0 } merkletree-master/merkletree.go0000644000175000017500000000242613152305166015626 0ustar freefree// Package merkletree provides tools for calculating the Merkle root of a // dataset, for creating a proof that a piece of data is in a Merkle tree of a // given root, and for verifying proofs that a piece of data is in a Merkle // tree of a given root. The tree is implemented according to the specification // for Merkle trees provided in RFC 6962. // // Package merkletree also supports building roots and proofs from cached // subroots of the Merkle tree. For example, a large file could be cached by // building the Merkle root for each 4MB sector and remembering the Merkle // roots of each sector. Using a cached tree, the Merkle root of the whole file // can be computed by passing the cached tree each of the roots of the 4MB // sector. Building proofs using these cached roots is also supported. A proof // must be built within the target sector using a normal Tree, requiring the // whole sector to be hashed. The results of that proof can then be passed into // the Prove() function of a cached tree, which will create the full proof // without needing to hash the entire file. Caching also makes it inexpensive // to update the Merkle root of the file after changing or deleting segments of // the larger file. // // Examples can be found in the README for the package. package merkletree merkletree-master/readers.go0000644000175000017500000000435013152305166015112 0ustar freefreepackage merkletree import ( "errors" "hash" "io" ) // ReadAll will read segments of size 'segmentSize' and push them into the tree // until EOF is reached. Success will return 'err == nil', not 'err == EOF'. No // padding is added to the data, so the last element may be smaller than // 'segmentSize'. func (t *Tree) ReadAll(r io.Reader, segmentSize int) error { for { segment := make([]byte, segmentSize) n, readErr := io.ReadFull(r, segment) if readErr == io.EOF { // All data has been read. break } else if readErr == io.ErrUnexpectedEOF { // This is the last segment, and there aren't enough bytes to fill // the entire segment. Note that the next call will return io.EOF. segment = segment[:n] } else if readErr != nil { return readErr } t.Push(segment) } return nil } // ReaderRoot returns the Merkle root of the data read from the reader, where // each leaf is 'segmentSize' long and 'h' is used as the hashing function. All // leaves will be 'segmentSize' bytes except the last leaf, which will not be // padded out if there are not enough bytes remaining in the reader. func ReaderRoot(r io.Reader, h hash.Hash, segmentSize int) (root []byte, err error) { tree := New(h) err = tree.ReadAll(r, segmentSize) if err != nil { return } root = tree.Root() return } // BuildReaderProof returns a proof that certain data is in the merkle tree // created by the data in the reader. The merkle root, set of proofs, and the // number of leaves in the Merkle tree are all returned. All leaves will we // 'segmentSize' bytes except the last leaf, which will not be padded out if // there are not enough bytes remaining in the reader. func BuildReaderProof(r io.Reader, h hash.Hash, segmentSize int, index uint64) (root []byte, proofSet [][]byte, numLeaves uint64, err error) { tree := New(h) err = tree.SetIndex(index) if err != nil { // This code should be unreachable - SetIndex will only return an error // if the tree is not empty, and yet the tree should be empty at this // point. panic(err) } err = tree.ReadAll(r, segmentSize) if err != nil { return } root, proofSet, _, numLeaves = tree.Prove() if len(proofSet) == 0 { err = errors.New("index was not reached while creating proof") return } return } merkletree-master/readers_test.go0000644000175000017500000000615513152305166016156 0ustar freefreepackage merkletree import ( "bytes" "crypto/sha256" "testing" ) // TestReaderRoot calls ReaderRoot on a manually crafted dataset // and checks the output. func TestReaderRoot(t *testing.T) { mt := CreateMerkleTester(t) bytes8 := []byte{0, 1, 2, 3, 4, 5, 6, 7} reader := bytes.NewReader(bytes8) root, err := ReaderRoot(reader, sha256.New(), 1) if err != nil { t.Fatal(err) } if bytes.Compare(root, mt.roots[8]) != 0 { t.Error("ReaderRoot returned the wrong root") } } // TestReaderRootPadding passes ReaderRoot a reader that has too few bytes to // fill the last segment. The segment should not be padded out. func TestReaderRootPadding(t *testing.T) { bytes1 := []byte{1} reader := bytes.NewReader(bytes1) root, err := ReaderRoot(reader, sha256.New(), 2) if err != nil { t.Fatal(err) } expectedRoot := sum(sha256.New(), []byte{0, 1}) if bytes.Compare(root, expectedRoot) != 0 { t.Error("ReaderRoot returned the wrong root") } bytes3 := []byte{1, 2, 3} reader = bytes.NewReader(bytes3) root, err = ReaderRoot(reader, sha256.New(), 2) if err != nil { t.Fatal(err) } baseLeft := sum(sha256.New(), []byte{0, 1, 2}) baseRight := sum(sha256.New(), []byte{0, 3}) expectedRoot = sum(sha256.New(), append(append([]byte{1}, baseLeft...), baseRight...)) if bytes.Compare(root, expectedRoot) != 0 { t.Error("ReaderRoot returned the wrong root") } } // TestBuildReaderProof calls BuildReaderProof on a manually crafted dataset // and checks the output. func TestBuilReaderProof(t *testing.T) { mt := CreateMerkleTester(t) bytes7 := []byte{0, 1, 2, 3, 4, 5, 6} reader := bytes.NewReader(bytes7) root, proofSet, numLeaves, err := BuildReaderProof(reader, sha256.New(), 1, 5) if err != nil { t.Fatal(err) } if bytes.Compare(root, mt.roots[7]) != 0 { t.Error("BuildReaderProof returned the wrong root") } if len(proofSet) != len(mt.proofSets[7][5]) { t.Fatal("BuildReaderProof returned a proof with the wrong length") } for i := range proofSet { if bytes.Compare(proofSet[i], mt.proofSets[7][5][i]) != 0 { t.Error("BuildReaderProof returned an incorrect proof") } } if numLeaves != 7 { t.Error("BuildReaderProof returned the wrong number of leaves") } } // TestBuildReaderProofPadding passes BuildReaderProof a reader that has too // few bytes to fill the last segment. The segment should not be padded out. func TestBuildReaderProofPadding(t *testing.T) { bytes1 := []byte{1} reader := bytes.NewReader(bytes1) root, proofSet, numLeaves, err := BuildReaderProof(reader, sha256.New(), 2, 0) if err != nil { t.Fatal(err) } expectedRoot := sum(sha256.New(), []byte{0, 1}) if bytes.Compare(root, expectedRoot) != 0 { t.Error("ReaderRoot returned the wrong root") } if len(proofSet) != 1 { t.Fatal("proofSet is the incorrect length") } if bytes.Compare(proofSet[0], []byte{1}) != 0 { t.Error("proofSet is incorrect") } if numLeaves != 1 { t.Error("wrong number of leaves returned") } } // TestEmptyReader passes an empty reader into BuildReaderProof. func TestEmptyReader(t *testing.T) { _, _, _, err := BuildReaderProof(new(bytes.Reader), sha256.New(), 64, 5) if err == nil { t.Error(err) } } merkletree-master/tree.go0000644000175000017500000002353213152305166014427 0ustar freefreepackage merkletree import ( "errors" "hash" ) // A Tree takes data as leaves and returns the Merkle root. Each call to 'Push' // adds one leaf to the Merkle tree. Calling 'Root' returns the Merkle root. // The Tree also constructs proof that a single leaf is a part of the tree. The // leaf can be chosen with 'SetIndex'. The memory footprint of Tree grows in // O(log(n)) in the number of leaves. type Tree struct { // The Tree is stored as a stack of subtrees. Each subtree has a height, // and is the Merkle root of 2^height leaves. A Tree with 11 nodes is // represented as a subtree of height 3 (8 nodes), a subtree of height 1 (2 // nodes), and a subtree of height 0 (1 node). Head points to the smallest // tree. When a new leaf is inserted, it is inserted as a subtree of height // 0. If there is another subtree of the same height, both can be removed, // combined, and then inserted as a subtree of height n + 1. head *subTree hash hash.Hash // Helper variables used to construct proofs that the data at 'proofIndex' // is in the Merkle tree. The proofSet is constructed as elements are being // added to the tree. The first element of the proof set is the original // data used to create the leaf at index 'proofIndex'. currentIndex uint64 proofIndex uint64 proofSet [][]byte // The cachedTree flag indicates that the tree is cached, meaning that // different code is used in 'Push' for creating a new head subtree. Adding // this flag is somewhat gross, but eliminates needing to duplicate the // entire 'Push' function when writing the cached tree. cachedTree bool } // A subTree contains the Merkle root of a complete (2^height leaves) subTree // of the Tree. 'sum' is the Merkle root of the subTree. If 'next' is not nil, // it will be a tree with a higher height. type subTree struct { next *subTree height int // Int is okay because a height over 300 is physically unachievable. sum []byte } // sum returns the hash of the input data using the specified algorithm. func sum(h hash.Hash, data ...[]byte) []byte { h.Reset() for _, d := range data { // the Hash interface specifies that Write never returns an error _, _ = h.Write(d) } return h.Sum(nil) } // leafSum returns the hash created from data inserted to form a leaf. Leaf // sums are calculated using: // Hash(0x00 || data) func leafSum(h hash.Hash, data []byte) []byte { return sum(h, []byte{0}, data) } // nodeSum returns the hash created from two sibling nodes being combined into // a parent node. Node sums are calculated using: // Hash(0x01 || left sibling sum || right sibling sum) func nodeSum(h hash.Hash, a, b []byte) []byte { return sum(h, []byte{1}, a, b) } // joinSubTrees combines two equal sized subTrees into a larger subTree. func joinSubTrees(h hash.Hash, a, b *subTree) *subTree { if DEBUG { if b.next != a { panic("invalid subtree join - 'a' is not paired with 'b'") } if a.height < b.height { panic("invalid subtree presented - height mismatch") } } return &subTree{ next: a.next, height: a.height + 1, sum: nodeSum(h, a.sum, b.sum), } } // New creates a new Tree. The provided hash will be used for all hashing // operations within the Tree. func New(h hash.Hash) *Tree { return &Tree{ hash: h, } } // Prove creates a proof that the leaf at the established index (established by // SetIndex) is an element of the Merkle tree. Prove will return a nil proof // set if used incorrectly. Prove does not modify the Tree. func (t *Tree) Prove() (merkleRoot []byte, proofSet [][]byte, proofIndex uint64, numLeaves uint64) { // Return nil if the Tree is empty, or if the proofIndex hasn't yet been // reached. if t.head == nil || len(t.proofSet) == 0 { return t.Root(), nil, t.proofIndex, t.currentIndex } proofSet = t.proofSet // The set of subtrees must now be collapsed into a single root. The proof // set already contains all of the elements that are members of a complete // subtree. Of what remains, there will be at most 1 element provided from // a sibling on the right, and all of the other proofs will be provided // from a sibling on the left. This results from the way orphans are // treated. All subtrees smaller than the subtree containing the proofIndex // will be combined into a single subtree that gets combined with the // proofIndex subtree as a single right sibling. All subtrees larger than // the subtree containing the proofIndex will be combined with the subtree // containing the proof index as left siblings. // Start at the smallest subtree and combine it with larger subtrees until // it would be combining with the subtree that contains the proof index. We // can recognize the subtree containing the proof index because the height // of that subtree will be one less than the current length of the proof // set. current := t.head for current.next != nil && current.next.height < len(proofSet)-1 { current = joinSubTrees(t.hash, current.next, current) } // Sanity check - check that either 'current' or 'current.next' is the // subtree containing the proof index. if DEBUG { if current.height != len(t.proofSet)-1 && (current.next != nil && current.next.height != len(t.proofSet)-1) { panic("could not find the subtree containing the proof index") } } // If the current subtree is not the subtree containing the proof index, // then it must be an aggregate subtree that is to the right of the subtree // containing the proof index, and the next subtree is the subtree // containing the proof index. if current.next != nil && current.next.height == len(proofSet)-1 { proofSet = append(proofSet, current.sum) current = current.next } // The current subtree must be the subtree containing the proof index. This // subtree does not need an entry, as the entry was created during the // construction of the Tree. Instead, skip to the next subtree. current = current.next // All remaining subtrees will be added to the proof set as a left sibling, // completing the proof set. for current != nil { proofSet = append(proofSet, current.sum) current = current.next } return t.Root(), proofSet, t.proofIndex, t.currentIndex } // Push will add data to the set, building out the Merkle tree and Root. The // tree does not remember all elements that are added, instead only keeping the // log(n) elements that are necessary to build the Merkle root and keeping the // log(n) elements necessary to build a proof that a piece of data is in the // Merkle tree. func (t *Tree) Push(data []byte) { // The first element of a proof is the data at the proof index. If this // data is being inserted at the proof index, it is added to the proof set. if t.currentIndex == t.proofIndex { t.proofSet = append(t.proofSet, data) } // Hash the data to create a subtree of height 0. The sum of the new node // is going to be the data for cached trees, and is going to be the result // of calling leafSum() on the data for standard trees. Doing a check here // prevents needing to duplicate the entire 'Push' function for the trees. t.head = &subTree{ next: t.head, height: 0, } if t.cachedTree { t.head.sum = data } else { t.head.sum = leafSum(t.hash, data) } // Insert the subTree into the Tree. As long as the height of the next // subTree is the same as the height of the current subTree, the two will // be combined into a single subTree of height n+1. for t.head.next != nil && t.head.height == t.head.next.height { // Before combining subtrees, check whether one of the subtree hashes // needs to be added to the proof set. This is going to be true IFF the // subtrees being combined are one height higher than the previous // subtree added to the proof set. The height of the previous subtree // added to the proof set is equal to len(t.proofSet) - 1. if t.head.height == len(t.proofSet)-1 { // One of the subtrees needs to be added to the proof set. The // subtree that needs to be added is the subtree that does not // contain the proofIndex. Because the subtrees being compared are // the smallest and rightmost trees in the Tree, this can be // determined by rounding the currentIndex down to the number of // nodes in the subtree and comparing that index to the proofIndex. leaves := uint64(1 << uint(t.head.height)) mid := (t.currentIndex / leaves) * leaves if t.proofIndex < mid { t.proofSet = append(t.proofSet, t.head.sum) } else { t.proofSet = append(t.proofSet, t.head.next.sum) } // Sanity check - the proofIndex should never be less than the // midpoint minus the number of leaves in each subtree. if DEBUG { if t.proofIndex < mid-leaves { panic("proof being added with weird values") } } } // Join the two subTrees into one subTree with a greater height. Then // compare the new subTree to the next subTree. t.head = joinSubTrees(t.hash, t.head.next, t.head) } t.currentIndex++ // Sanity check - From head to tail of the stack, the height should be // strictly increasing. if DEBUG { current := t.head height := current.height for current.next != nil { current = current.next if current.height <= height { panic("subtrees are out of order") } height = current.height } } } // Root returns the Merkle root of the data that has been pushed. func (t *Tree) Root() []byte { // If the Tree is empty, return nil. if t.head == nil { return nil } // The root is formed by hashing together subTrees in order from least in // height to greatest in height. The taller subtree is the first subtree in // the join. current := t.head for current.next != nil { current = joinSubTrees(t.hash, current.next, current) } return current.sum } // SetIndex will tell the Tree to create a storage proof for the leaf at the // input index. SetIndex must be called on an empty tree. func (t *Tree) SetIndex(i uint64) error { if t.head != nil { return errors.New("cannot call SetIndex on Tree if Tree has not been reset") } t.proofIndex = i return nil } merkletree-master/tree_test.go0000644000175000017500000003273113152305166015467 0ustar freefreepackage merkletree import ( "bytes" "crypto/rand" "crypto/sha256" "math/big" "strconv" "testing" ) // A MerkleTester contains data types that can be filled out manually to // compare against function results. type MerkleTester struct { // data is the raw data of the Merkle tree. data [][]byte // leaves is the hashes of the data, and should be the same length. leaves [][]byte // roots contains the root hashes of Merkle trees of various heights using // the data for input. roots map[int][]byte // proofSets contains proofs that certain data is in a Merkle tree. The // first map is the number of leaves in the tree that the proof is for. The // root of that tree can be found in roots. The second map is the // proofIndex that was used when building the proof. proofSets map[int]map[int][][]byte *testing.T } // join returns the sha256 hash of 0x01 || a || b. func (mt *MerkleTester) join(a, b []byte) []byte { return sum(sha256.New(), append(append([]byte{1}, a...), b...)) } // CreateMerkleTester creates a Merkle tester and manually fills out many of // the expected values for constructing Merkle tree roots and Merkle tree // proofs. These manual values can then be compared against the values that the // Tree creates. func CreateMerkleTester(t *testing.T) (mt *MerkleTester) { mt = &MerkleTester{ roots: make(map[int][]byte), proofSets: make(map[int]map[int][][]byte), } mt.T = t // Fill out the data and leaves values. size := 16 for i := 0; i < size; i++ { mt.data = append(mt.data, []byte{byte(i)}) } for i := 0; i < size; i++ { mt.leaves = append(mt.leaves, sum(sha256.New(), append([]byte{0}, mt.data[i]...))) } // Manually build out expected Merkle root values. mt.roots[0] = nil mt.roots[1] = mt.leaves[0] mt.roots[2] = mt.join(mt.leaves[0], mt.leaves[1]) mt.roots[3] = mt.join( mt.roots[2], mt.leaves[2], ) mt.roots[4] = mt.join( mt.roots[2], mt.join(mt.leaves[2], mt.leaves[3]), ) mt.roots[5] = mt.join( mt.roots[4], mt.leaves[4], ) mt.roots[6] = mt.join( mt.roots[4], mt.join( mt.leaves[4], mt.leaves[5], ), ) mt.roots[7] = mt.join( mt.roots[4], mt.join( mt.join(mt.leaves[4], mt.leaves[5]), mt.leaves[6], ), ) mt.roots[8] = mt.join( mt.roots[4], mt.join( mt.join(mt.leaves[4], mt.leaves[5]), mt.join(mt.leaves[6], mt.leaves[7]), ), ) mt.roots[15] = mt.join( mt.roots[8], mt.join( mt.join( mt.join(mt.leaves[8], mt.leaves[9]), mt.join(mt.leaves[10], mt.leaves[11]), ), mt.join( mt.join(mt.leaves[12], mt.leaves[13]), mt.leaves[14], ), ), ) // Manually build out some proof sets that should should match what the // Tree creates for the same values. mt.proofSets[1] = make(map[int][][]byte) mt.proofSets[1][0] = [][]byte{mt.data[0]} mt.proofSets[2] = make(map[int][][]byte) mt.proofSets[2][0] = [][]byte{ mt.data[0], mt.leaves[1], } mt.proofSets[2][1] = [][]byte{ mt.data[1], mt.leaves[0], } mt.proofSets[5] = make(map[int][][]byte) mt.proofSets[5][4] = [][]byte{ mt.data[4], mt.roots[4], } mt.proofSets[6] = make(map[int][][]byte) mt.proofSets[6][0] = [][]byte{ mt.data[0], mt.leaves[1], mt.join( mt.leaves[2], mt.leaves[3], ), mt.join( mt.leaves[4], mt.leaves[5], ), } mt.proofSets[6][2] = [][]byte{ mt.data[2], mt.leaves[3], mt.roots[2], mt.join( mt.leaves[4], mt.leaves[5], ), } mt.proofSets[6][4] = [][]byte{ mt.data[4], mt.leaves[5], mt.roots[4], } mt.proofSets[6][5] = [][]byte{ mt.data[5], mt.leaves[4], mt.roots[4], } mt.proofSets[7] = make(map[int][][]byte) mt.proofSets[7][5] = [][]byte{ mt.data[5], mt.leaves[4], mt.leaves[6], mt.roots[4], } mt.proofSets[15] = make(map[int][][]byte) mt.proofSets[15][3] = [][]byte{ mt.data[3], mt.leaves[2], mt.roots[2], mt.join( mt.join(mt.leaves[4], mt.leaves[5]), mt.join(mt.leaves[6], mt.leaves[7]), ), mt.join( mt.join( mt.join(mt.leaves[8], mt.leaves[9]), mt.join(mt.leaves[10], mt.leaves[11]), ), mt.join( mt.join(mt.leaves[12], mt.leaves[13]), mt.leaves[14], ), ), } mt.proofSets[15][10] = [][]byte{ mt.data[10], mt.leaves[11], mt.join( mt.leaves[8], mt.leaves[9], ), mt.join( mt.join(mt.leaves[12], mt.leaves[13]), mt.leaves[14], ), mt.roots[8], } mt.proofSets[15][13] = [][]byte{ mt.data[13], mt.leaves[12], mt.leaves[14], mt.join( mt.join(mt.leaves[8], mt.leaves[9]), mt.join(mt.leaves[10], mt.leaves[11]), ), mt.roots[8], } return } // TestBuildRoot checks that the root returned by Tree matches the manually // created roots for all of the manually created roots. func TestBuildRoot(t *testing.T) { mt := CreateMerkleTester(t) // Compare the results of calling Root against all of the manually // constructed Merkle trees. var tree *Tree for i, root := range mt.roots { // Fill out the tree. tree = New(sha256.New()) for j := 0; j < i; j++ { tree.Push(mt.data[j]) } // Get the root and compare to the manually constructed root. treeRoot := tree.Root() if bytes.Compare(root, treeRoot) != 0 { t.Error("tree root doesn't match manual root for index", i) } } } // TestBuildAndVerifyProof builds a proof using a tree for every single // manually created proof in the MerkleTester. Then it checks that the proof // matches the manually created proof, and that the proof is verified by // VerifyProof. Then it checks that the proof fails for all other indices, // which should happen if all of the leaves are unique. func TestBuildAndVerifyProof(t *testing.T) { mt := CreateMerkleTester(t) // Compare the results of building a Merkle proof to all of the manually // constructed proofs. tree := New(sha256.New()) for i, manualProveSets := range mt.proofSets { for j, expectedProveSet := range manualProveSets { // Build out the tree. tree = New(sha256.New()) err := tree.SetIndex(uint64(j)) if err != nil { t.Fatal(err) } for k := 0; k < i; k++ { tree.Push(mt.data[k]) } // Get the proof and check all values. merkleRoot, proofSet, proofIndex, numSegments := tree.Prove() if bytes.Compare(merkleRoot, mt.roots[i]) != 0 { t.Error("incorrect Merkle root returned by Tree for indices", i, j) } if len(proofSet) != len(expectedProveSet) { t.Error("proof set is wrong length for indices", i, j) continue } if proofIndex != uint64(j) { t.Error("incorrect proofIndex returned for indices", i, j) } if numSegments != uint64(i) { t.Error("incorrect numSegments returned for indices", i, j) } for k := range proofSet { if bytes.Compare(proofSet[k], expectedProveSet[k]) != 0 { t.Error("proof set does not match expected proof set for indices", i, j, k) } } // Check that verification works on for the desired proof index but // fails for all other indices. if !VerifyProof(sha256.New(), merkleRoot, proofSet, proofIndex, numSegments) { t.Error("proof set does not verify for indices", i, j) } for k := uint64(0); k < uint64(i); k++ { if k == proofIndex { continue } if VerifyProof(sha256.New(), merkleRoot, proofSet, k, numSegments) { t.Error("proof set verifies for wrong index at indices", i, j, k) } } // Check that calling Prove a second time results in the same // values. merkleRoot2, proofSet2, proofIndex2, numSegments2 := tree.Prove() if bytes.Compare(merkleRoot, merkleRoot2) != 0 { t.Error("tree returned different merkle roots after calling Prove twice for indices", i, j) } if len(proofSet) != len(proofSet2) { t.Error("tree returned different proof sets after calling Prove twice for indices", i, j) } for k := range proofSet { if bytes.Compare(proofSet[k], proofSet2[k]) != 0 { t.Error("tree returned different proof sets after calling Prove twice for indices", i, j) } } if proofIndex != proofIndex2 { t.Error("tree returned different proof indexes after calling Prove twice for indices", i, j) } if numSegments != numSegments2 { t.Error("tree returned different segment count after calling Prove twice for indices", i, j) } } } } // TestBadInputs provides malicious inputs to the functions of the package, // trying to trigger panics or unexpected behavior. func TestBadInputs(t *testing.T) { // Get the root and proof of an empty tree. tree := New(sha256.New()) root := tree.Root() if root != nil { t.Error("root of empty tree should be nil") } _, proof, _, _ := tree.Prove() if proof != nil { t.Error("proof of empty tree should be nil") } // Get the proof of a tree that hasn't reached it's index. err := tree.SetIndex(3) if err != nil { t.Fatal(err) } tree.Push([]byte{1}) _, proof, _, _ = tree.Prove() if proof != nil { t.Fatal(err) } err = tree.SetIndex(2) if err == nil { t.Error("expecting error, shouldn't be able to reset a tree after pushing") } // Try nil values in VerifyProof. mt := CreateMerkleTester(t) if VerifyProof(sha256.New(), nil, mt.proofSets[1][0], 0, 1) { t.Error("VerifyProof should return false for nil merkle root") } if VerifyProof(sha256.New(), []byte{1}, nil, 0, 1) { t.Error("VerifyProof should return false for nil proof set") } if VerifyProof(sha256.New(), mt.roots[15], mt.proofSets[15][3][1:], 3, 15) { t.Error("VerifyProof should return false for too-short proof set") } if VerifyProof(sha256.New(), mt.roots[15], mt.proofSets[15][10][1:], 10, 15) { t.Error("VerifyProof should return false for too-short proof set") } if VerifyProof(sha256.New(), mt.roots[15], mt.proofSets[15][10], 15, 0) { t.Error("VerifyProof should return false when numLeaves is 0") } } // TestCompatibility runs BuildProof for a large set of trees, and checks that // verify affirms each proof, while rejecting for all other indexes (this // second half requires that all input data be unique). The test checks that // build and verify are internally consistent, but doesn't check for actual // correctness. func TestCompatibility(t *testing.T) { if testing.Short() { t.SkipNow() } // Brute force all trees up to size 'max'. Running time for this test is max^3. max := uint64(129) tree := New(sha256.New()) for i := uint64(1); i < max; i++ { // Try with proof at every possible index. for j := uint64(0); j < i; j++ { // Push unique data into the tree. tree = New(sha256.New()) err := tree.SetIndex(j) if err != nil { t.Fatal(err) } for k := uint64(0); k < i; k++ { tree.Push([]byte{byte(k)}) } // Build the proof for the tree and run it through verify. merkleRoot, proofSet, proofIndex, numLeaves := tree.Prove() if !VerifyProof(sha256.New(), merkleRoot, proofSet, proofIndex, numLeaves) { t.Error("proof didn't verify for indices", i, j) } // Check that verification fails for all other indices. for k := uint64(0); k < i; k++ { if k == j { continue } if VerifyProof(sha256.New(), merkleRoot, proofSet, k, numLeaves) { t.Error("proof verified for indices", i, j, k) } } } } // Check that proofs on larger trees are consistent. for i := 0; i < 25; i++ { // Determine a random size for the tree up to 64M elements. sizeI, err := rand.Int(rand.Reader, big.NewInt(256e3)) if err != nil { t.Fatal(err) } size := uint64(sizeI.Int64()) proofIndexI, err := rand.Int(rand.Reader, sizeI) if err != nil { t.Fatal(err) } proofIndex := uint64(proofIndexI.Int64()) // Prepare the tree. tree = New(sha256.New()) err = tree.SetIndex(proofIndex) if err != nil { t.Fatal(err) } // Insert 'size' unique elements. for j := 0; j < int(size); j++ { elem := []byte(strconv.Itoa(j)) tree.Push(elem) } // Get the proof for the tree and run it through verify. merkleRoot, proofSet, proofIndex, numLeaves := tree.Prove() if !VerifyProof(sha256.New(), merkleRoot, proofSet, proofIndex, numLeaves) { t.Error("proof didn't verify in long test", size, proofIndex) } } } // TestLeafCounts checks that the number of leaves in the tree are being // reported correctly. func TestLeafCounts(t *testing.T) { tree := New(sha256.New()) err := tree.SetIndex(0) if err != nil { t.Fatal(err) } _, _, _, leaves := tree.Prove() if leaves != 0 { t.Error("bad reporting of leaf count") } tree = New(sha256.New()) err = tree.SetIndex(0) if err != nil { t.Fatal(err) } tree.Push([]byte{}) _, _, _, leaves = tree.Prove() if leaves != 1 { t.Error("bad reporting on leaf count") } } // BenchmarkSha256_4MB uses sha256 to hash 4mb of data. func BenchmarkSha256_4MB(b *testing.B) { data := make([]byte, 4*1024*1024) _, err := rand.Read(data) if err != nil { b.Fatal(err) } b.ResetTimer() for i := 0; i < b.N; i++ { sha256.Sum256(data) } } // BenchmarkTree64_4MB creates a Merkle tree out of 4MB using a segment size of // 64 bytes, using sha256. func BenchmarkTree64_4MB(b *testing.B) { data := make([]byte, 4*1024*1024) _, err := rand.Read(data) if err != nil { b.Fatal(err) } segmentSize := 64 b.ResetTimer() tree := New(sha256.New()) for i := 0; i < b.N; i++ { for j := 0; j < len(data)/segmentSize; j++ { tree.Push(data[j*segmentSize : (j+1)*segmentSize]) } tree.Root() } } // BenchmarkTree4k_4MB creates a Merkle tree out of 4MB using a segment size of // 4096 bytes, using sha256. func BenchmarkTree4k_4MB(b *testing.B) { data := make([]byte, 4*1024*1024) _, err := rand.Read(data) if err != nil { b.Fatal(err) } segmentSize := 4096 b.ResetTimer() tree := New(sha256.New()) for i := 0; i < b.N; i++ { for j := 0; j < len(data)/segmentSize; j++ { tree.Push(data[j*segmentSize : (j+1)*segmentSize]) } tree.Root() } } merkletree-master/verify.go0000644000175000017500000001123713152305166014773 0ustar freefreepackage merkletree import ( "bytes" "hash" ) // VerifyProof takes a Merkle root, a proofSet, and a proofIndex and returns // true if the first element of the proof set is a leaf of data in the Merkle // root. False is returned if the proof set or Merkle root is nil, and if // 'numLeaves' equals 0. func VerifyProof(h hash.Hash, merkleRoot []byte, proofSet [][]byte, proofIndex uint64, numLeaves uint64) bool { // Return false for nonsense input. A switch statement is used so that the // cover tool will reveal if a case is not covered by the test suite. This // would not be possible using a single if statement due to the limitations // of the cover tool. if merkleRoot == nil { return false } if proofIndex >= numLeaves { return false } // In a Merkle tree, every node except the root node has a sibling. // Combining the two siblings in the correct order will create the parent // node. Each of the remaining hashes in the proof set is a sibling to a // node that can be built from all of the previous elements of the proof // set. The next node is built by taking: // // H(0x01 || sibling A || sibling B) // // The difficulty of the algorithm lies in determining whether the supplied // hash is sibling A or sibling B. This information can be determined by // using the proof index and the total number of leaves in the tree. // // A pair of two siblings forms a subtree. The subtree is complete if it // has 1 << height total leaves. When the subtree is complete, the position // of the proof index within the subtree can be determined by looking at // the bounds of the subtree and determining if the proof index is in the // first or second half of the subtree. // // When the subtree is not complete, either 1 or 0 of the remaining hashes // will be sibling B. All remaining hashes after that will be sibling A. // This is true because of the way that orphans are merged into the Merkle // tree - an orphan at height n is elevated to height n + 1, and only // hashed when it is no longer an orphan. Each subtree will therefore merge // with at most 1 orphan to the right before becoming an orphan itself. // Orphan nodes are always merged with larger subtrees to the left. // // One vulnerability with the proof verification is that the proofSet may // not be long enough. Before looking at an element of proofSet, a check // needs to be made that the element exists. // The first element of the set is the original data. A sibling at height 1 // is created by getting the leafSum of the original data. height := 0 if len(proofSet) <= height { return false } sum := leafSum(h, proofSet[height]) height++ // While the current subtree (of height 'height') is complete, determine // the position of the next sibling using the complete subtree algorithm. // 'stableEnd' tells us the ending index of the last full subtree. It gets // initialized to 'proofIndex' because the first full subtree was the // subtree of height 1, created above (and had an ending index of // 'proofIndex'). stableEnd := proofIndex for { // Determine if the subtree is complete. This is accomplished by // rounding down the proofIndex to the nearest 1 << 'height', adding 1 // << 'height', and comparing the result to the number of leaves in the // Merkle tree. subTreeStartIndex := (proofIndex / (1 << uint(height))) * (1 << uint(height)) // round down to the nearest 1 << height subTreeEndIndex := subTreeStartIndex + (1 << (uint(height))) - 1 // subtract 1 because the start index is inclusive if subTreeEndIndex >= numLeaves { // If the Merkle tree does not have a leaf at index // 'subTreeEndIndex', then the subtree of the current height is not // a complete subtree. break } stableEnd = subTreeEndIndex // Determine if the proofIndex is in the first or the second half of // the subtree. if len(proofSet) <= height { return false } if proofIndex-subTreeStartIndex < 1<