pax_global_header00006660000000000000000000000064141372563140014520gustar00rootroot0000000000000052 comment=09cb8ecd1b8d70d543be74df50e41ad3cd4f0b01 blobloom-0.7.1/000077500000000000000000000000001413725631400133325ustar00rootroot00000000000000blobloom-0.7.1/.golangci.yml000066400000000000000000000005141413725631400157160ustar00rootroot00000000000000# Configuration for golangci-lint. linters: disable: - asciicheck enable: - gocognit - gocyclo - godot - gofumpt - lll - misspell - nakedret - thelper issues: exclude-rules: - path: _test\.go linters: errcheck linters-settings: govet: enable: - atomicalign blobloom-0.7.1/LICENSE000066400000000000000000000261361413725631400143470ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. blobloom-0.7.1/README.md000066400000000000000000000057111413725631400146150ustar00rootroot00000000000000Blobloom ======== A Bloom filter package for Go (golang) with no compile-time dependencies. This package implements a version of Bloom filters called [blocked Bloom filters]( https://algo2.iti.kit.edu/documents/cacheefficientbloomfilters-jea.pdf), which get a speed boost from using the CPU cache more efficiently than regular Bloom filters. Unlike most Bloom filter packages for Go, this one doesn't run a hash function for you. That's a benefit if you need a custom hash or you want pick the fastest one for an application. Usage ----- To construct a Bloom filter, you need to know how many keys you want to store and what rate of false positives you find acceptable. f := blobloom.NewOptimized(blobloom.Config{ Capacity: nkeys, // Expected number of keys. FPRate: 1e-4, // Accept one false positive per 10,000 lookups. }) To add a key: // import "github.com/cespare/xxhash/v2" f.Add(xxhash.Sum64(key)) To test for the presence of a key in the filter: if f.Has(xxhash.Sum64(key)) { // Key is probably in f. } else { // Key is certainly in f. } The false positive rate is defined as usual: if you look up 10,000 random keys in a Bloom filter filled to capacity, an expected one of those is a false positive for FPRate 1e-4. See the examples/ directory and the [package documentation](https://pkg.go.dev/github.com/greatroar/blobloom) for further usage information and examples. Hash functions -------------- Blobloom does not provide hash functions. Instead, it requires client code to represent each key as a single 64-bit hash value, leaving it to the user to pick the right hash function for a particular problem. Here are some general suggestions: * If you use Bloom filters to speed up access to a key-value store, you might want to look at [xxh3](https://github.com/zeebo/xxh3) or [xxhash]( https://github.com/cespare/xxhash). * If your keys are cryptographic hashes, consider using the first 8 bytes of those hashes. * If you use Bloom filters to make probabilistic decisions, a randomized hash function such as [maphash](https://golang.org/pkg/hash/maphash) should prevent the same false positives occurring every time. When evaluating a hash function, or designing a custom one, make sure it is a 64-bit hash that properly mixes its input bits. Casting a 32-bit hash to uint64 gives suboptimal results. So does passing integer keys in without running them through a mixing function. License ------- Copyright © 2020-2021 the Blobloom authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. blobloom-0.7.1/benchmark_test.go000066400000000000000000000061701413725631400166560ustar00rootroot00000000000000// Copyright 2020 the Blobloom authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Benchmarks for the basic operations live in the benchmarks/ subpackage. package blobloom import ( "math/rand" "sync" "sync/atomic" "testing" ) // Baseline for BenchmarkAddSync. func benchmarkAddLocked(b *testing.B, nbits uint64) { b.Helper() const nhashes = 22 // Large number of hashes to create collisions. var ( f = New(nbits, nhashes) mu sync.Mutex seed uint32 ) b.ResetTimer() b.RunParallel(func(pb *testing.PB) { r := rand.New(rand.NewSource(int64(atomic.AddUint32(&seed, 1)))) for pb.Next() { mu.Lock() f.Add(r.Uint64()) mu.Unlock() } }) } func BenchmarkAddLocked128kB(b *testing.B) { benchmarkAddLocked(b, 1<<20) } func BenchmarkAddLocked1MB(b *testing.B) { benchmarkAddLocked(b, 1<<23) } func BenchmarkAddLocked16MB(b *testing.B) { benchmarkAddLocked(b, 1<<27) } func benchmarkAddSync(b *testing.B, nbits uint64) { b.Helper() const nhashes = 22 // Large number of hashes to create collisions. f := NewSync(nbits, nhashes) var seed uint32 b.ResetTimer() b.RunParallel(func(pb *testing.PB) { r := rand.New(rand.NewSource(int64(atomic.AddUint32(&seed, 1)))) for pb.Next() { f.Add(r.Uint64()) } }) } func BenchmarkAddSync128kB(b *testing.B) { benchmarkAddSync(b, 1<<20) } func BenchmarkAddSync1MB(b *testing.B) { benchmarkAddSync(b, 1<<23) } func BenchmarkAddSync16MB(b *testing.B) { benchmarkAddSync(b, 1<<27) } func BenchmarkCardinalityDense(b *testing.B) { f := New(1<<20, 2) for i := range f.b { for j := range f.b[i] { f.b[i][j] = rand.Uint32() } } b.ResetTimer() for i := 0; i < b.N; i++ { f.Cardinality() } } func BenchmarkCardinalitySparse(b *testing.B) { f := New(1<<20, 2) for i := 0; i < len(f.b); i += 2 { for _, j := range []int{4, 8, 13} { f.b[i][j] = rand.Uint32() } } b.ResetTimer() for i := 0; i < b.N; i++ { f.Cardinality() } } func BenchmarkOnescount(b *testing.B) { var blk block for i := range blk { blk[i] = rand.Uint32() } b.ResetTimer() for i := 0; i < b.N; i++ { onescount(&blk) } } func BenchmarkUnion(b *testing.B) { const n = 1e6 var ( cfg = Config{Capacity: n, FPRate: 1e-5} f = NewOptimized(cfg) g = NewOptimized(cfg) fRef = NewOptimized(cfg) gRef = NewOptimized(cfg) hashes = randomU64(n, 0xcb6231119) ) b.Logf("NumBits = %d", f.NumBits()) for _, h := range hashes[:n/2] { fRef.Add(h) } for _, h := range hashes[n/2:] { gRef.Add(h) } b.ResetTimer() for i := 0; i < b.N; i++ { b.StopTimer() f.Clear() f.Union(fRef) g.Clear() g.Union(gRef) b.StartTimer() f.Union(g) } } blobloom-0.7.1/benchmarks/000077500000000000000000000000001413725631400154475ustar00rootroot00000000000000blobloom-0.7.1/benchmarks/README.md000066400000000000000000000033141413725631400167270ustar00rootroot00000000000000This module contains benchmarks for comparison against other Bloom filter packages. To run these benchmarks, pick a build tag from the following table: | Tag | Package | | -------- | ----------------------------------------------------------- | | (no tag) | This package with pre-hashed inputs | | bbloom | github.com/ipfs/bbloom | | boom | github.com/tylertreat/BoomFilters ("classic" Bloom filters) | | sync | This package's SyncFilter with pre-hashed inputs | | willf | github.com/bits-and-blooms/bloom (formerly willf/bloom) | | xxhash | This package + github.com/cespare/xxhash | | xxh3 | This package + github.com/zeebo/xxh3 | Then invoke go test as follows: go test -tags="$tag" -bench=. Omit -tags and its argument to run the benchmarks for Blobloom. These assume that the input keys (which are random strings) can be used as hashes without any processing. This reflects the original use case (in [Syncthing]( https://syncthing.net)) where SHA-256 hashes were stored in a Bloom filter. If this does not describe your use case, benchmark with the tag xxhash to run the keys through the [xxhash](https://github.com/cespare/xxhash) function. The benchmarks are set up to work with the benchstat tool. To compare Blobloom+xxh3 to bbloom, do go get golang.org/x/perf/cmd/benchstat go test -bench=. -count=5 -timeout=30m -tags "bbloom" | tee bbloom.bench go test -bench=. -count=5 -timeout=30m -tags "xx3" | tee xxh3.bench benchstat bbloom.bench xxh3.bench The sync benchmark only measures sequential performance. blobloom-0.7.1/benchmarks/bbloom.go000066400000000000000000000015361413725631400172550ustar00rootroot00000000000000// Copyright 2020 the Blobloom authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // +build bbloom package benchmarks import "github.com/ipfs/bbloom" type bloomFilter = bbloom.Bloom func newBF(capacity int, fpr float64) *bloomFilter { f, err := bbloom.New(float64(capacity), fpr) if err != nil { panic(err) } return (*bloomFilter)(f) } blobloom-0.7.1/benchmarks/benchmark_test.go000066400000000000000000000126751413725631400210020ustar00rootroot00000000000000// Copyright 2020 the Blobloom authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package benchmarks contains benchmarks for various Bloom filter // implementations, selected by build tags. See README.md for details. package benchmarks import ( "math/rand" "testing" ) const hashSize = 32 func makehashes(n int, seed int64) []byte { h := make([]byte, n*hashSize) r := rand.New(rand.NewSource(seed)) r.Read(h) return h } // In each iteration, add a SHA-256 into a Bloom filter with the given capacity // and desired FPR. func benchmarkAdd(b *testing.B, capacity int, fpr float64) { b.Helper() hashes := makehashes(b.N, 51251991517) f := newBF(capacity, fpr) b.ResetTimer() for i := 0; i < b.N; i++ { h := hashes[i*hashSize : (i+1)*hashSize] f.Add(h) } } func BenchmarkAdd1e5_1e2(b *testing.B) { benchmarkAdd(b, 1e5, 1e-2) } func BenchmarkAdd1e6_1e2(b *testing.B) { benchmarkAdd(b, 1e6, 1e-2) } func BenchmarkAdd1e7_1e2(b *testing.B) { benchmarkAdd(b, 1e7, 1e-2) } func BenchmarkAdd1e8_1e2(b *testing.B) { benchmarkAdd(b, 1e8, 1e-2) } func BenchmarkAdd1e5_1e3(b *testing.B) { benchmarkAdd(b, 1e5, 1e-3) } func BenchmarkAdd1e6_1e3(b *testing.B) { benchmarkAdd(b, 1e6, 1e-3) } func BenchmarkAdd1e7_1e3(b *testing.B) { benchmarkAdd(b, 1e7, 1e-3) } func BenchmarkAdd1e8_1e3(b *testing.B) { benchmarkAdd(b, 1e8, 1e-3) } // In each iteration, test for a SHA-256 in a Bloom filter with the given capacity // and desired FPR that has that SHA-256 added to it. func benchmarkTestPos(b *testing.B, capacity int, fpr float64) { b.Helper() const ntest = 8192 hashes := makehashes(ntest, 0x5128351a) f := newBF(capacity, fpr) for i := 0; i < capacity && i < ntest; i++ { h := hashes[i*hashSize : (i+1)*hashSize] f.Add(h) } for i := ntest; i < capacity; i++ { h := make([]byte, hashSize) f.Add(h) } b.ResetTimer() for i := 0; i < b.N; i++ { j := i % ntest h := hashes[j*hashSize : (j+1)*hashSize] if !f.Has(h) { b.Fatalf("%x added to Bloom filter but not retrieved", h) } } } func BenchmarkTestPos1e5_1e2(b *testing.B) { benchmarkTestPos(b, 1e5, 1e-2) } func BenchmarkTestPos1e6_1e2(b *testing.B) { benchmarkTestPos(b, 1e6, 1e-2) } func BenchmarkTestPos1e7_1e2(b *testing.B) { benchmarkTestPos(b, 1e7, 1e-2) } func BenchmarkTestPos1e8_1e2(b *testing.B) { benchmarkTestPos(b, 1e8, 1e-2) } func BenchmarkTestPos1e5_1e3(b *testing.B) { benchmarkTestPos(b, 1e5, 1e-3) } func BenchmarkTestPos1e6_1e3(b *testing.B) { benchmarkTestPos(b, 1e6, 1e-3) } func BenchmarkTestPos1e7_1e3(b *testing.B) { benchmarkTestPos(b, 1e7, 1e-3) } func BenchmarkTestPos1e8_1e3(b *testing.B) { benchmarkTestPos(b, 1e8, 1e-3) } // In each iteration, test for the presence of a SHA-256 in a filled Bloom filter // with the given capacity and desired FPR. func benchmarkTestNeg(b *testing.B, capacity int, fpr float64) { b.Helper() r := rand.New(rand.NewSource(0xae694)) f := newBF(capacity, fpr) h := make([]byte, hashSize) for i := 0; i < capacity; i++ { r.Read(h) f.Add(h) } // Make new hashes. Assume these are all distinct from the inserted ones. const ntest = 8192 hashes := makehashes(ntest, 562175) b.ResetTimer() fp := 0 for i := 0; i < b.N; i++ { j := i % ntest h := hashes[j*hashSize : (j+1)*hashSize] if f.Has(h) { fp++ } } b.Logf("false positive rate = %.3f%%", 100*float64(fp)/float64(b.N)) } func BenchmarkTestNeg1e5_1e2(b *testing.B) { benchmarkTestNeg(b, 1e5, 1e-2) } func BenchmarkTestNeg1e6_1e2(b *testing.B) { benchmarkTestNeg(b, 1e6, 1e-2) } func BenchmarkTestNeg1e7_1e2(b *testing.B) { benchmarkTestNeg(b, 1e7, 1e-2) } func BenchmarkTestNeg1e8_1e2(b *testing.B) { benchmarkTestNeg(b, 1e8, 1e-2) } func BenchmarkTestNeg1e5_1e3(b *testing.B) { benchmarkTestNeg(b, 1e5, 1e-3) } func BenchmarkTestNeg1e6_1e3(b *testing.B) { benchmarkTestNeg(b, 1e6, 1e-3) } func BenchmarkTestNeg1e7_1e3(b *testing.B) { benchmarkTestNeg(b, 1e7, 1e-3) } func BenchmarkTestNeg1e8_1e3(b *testing.B) { benchmarkTestNeg(b, 1e8, 1e-3) } // In each iteration, test for the presence of a SHA-256 in an empty Bloom filter // with the given capacity and desired FPR. func benchmarkTestEmpty(b *testing.B, capacity int, fpr float64) { b.Helper() const ntest = 65536 hashes := makehashes(ntest, 054271) f := newBF(capacity, fpr) b.ResetTimer() for i := 0; i < b.N; i++ { j := i % ntest f.Has(hashes[j*hashSize : (j+1)*hashSize]) } } func BenchmarkTestEmpty1e5_1e2(b *testing.B) { benchmarkTestEmpty(b, 1e5, 1e-2) } func BenchmarkTestEmpty1e6_1e2(b *testing.B) { benchmarkTestEmpty(b, 1e6, 1e-2) } func BenchmarkTestEmpty1e7_1e2(b *testing.B) { benchmarkTestEmpty(b, 1e7, 1e-2) } func BenchmarkTestEmpty1e8_1e2(b *testing.B) { benchmarkTestEmpty(b, 1e8, 1e-2) } func BenchmarkTestEmpty1e5_1e3(b *testing.B) { benchmarkTestEmpty(b, 1e5, 1e-3) } func BenchmarkTestEmpty1e6_1e3(b *testing.B) { benchmarkTestEmpty(b, 1e6, 1e-3) } func BenchmarkTestEmpty1e7_1e3(b *testing.B) { benchmarkTestEmpty(b, 1e7, 1e-3) } func BenchmarkTestEmpty1e8_1e3(b *testing.B) { benchmarkTestEmpty(b, 1e8, 1e-3) } blobloom-0.7.1/benchmarks/blobloom.go000066400000000000000000000022631413725631400176060ustar00rootroot00000000000000// Copyright 2020 the Blobloom authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // +build !bbloom,!boom,!devopsfaith,!sync,!willf,!xxh3,!xxhash package benchmarks import ( "encoding/binary" "github.com/greatroar/blobloom" ) type bloomFilter blobloom.Filter func (f *bloomFilter) Add(hash []byte) { h := binary.BigEndian.Uint64(hash[:8]) ((*blobloom.Filter)(f)).Add(h) } func (f *bloomFilter) Has(hash []byte) bool { h := binary.BigEndian.Uint64(hash[:8]) return ((*blobloom.Filter)(f)).Has(h) } func newBF(capacity int, fpr float64) *bloomFilter { f := blobloom.NewOptimized(blobloom.Config{ Capacity: uint64(capacity), FPRate: fpr, }) return (*bloomFilter)(f) } blobloom-0.7.1/benchmarks/blobloom_xxh3.go000066400000000000000000000021431413725631400205550ustar00rootroot00000000000000// Copyright 2020 the Blobloom authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // +build xxh3 package benchmarks import ( "github.com/greatroar/blobloom" "github.com/zeebo/xxh3" ) type bloomFilter blobloom.Filter func (f *bloomFilter) Add(hash []byte) { h := xxh3.Hash(hash) ((*blobloom.Filter)(f)).Add(h) } func (f *bloomFilter) Has(hash []byte) bool { h := xxh3.Hash(hash) return ((*blobloom.Filter)(f)).Has(h) } func newBF(capacity int, fpr float64) *bloomFilter { f := blobloom.NewOptimized(blobloom.Config{ Capacity: uint64(capacity), FPRate: fpr, }) return (*bloomFilter)(f) } blobloom-0.7.1/benchmarks/blobloom_xxhash.go000066400000000000000000000021101413725631400211600ustar00rootroot00000000000000// Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // +build xxhash package benchmarks import ( "github.com/cespare/xxhash/v2" "github.com/greatroar/blobloom" ) type bloomFilter blobloom.Filter func (f *bloomFilter) Add(hash []byte) { h := xxhash.Sum64(hash) ((*blobloom.Filter)(f)).Add(h) } func (f *bloomFilter) Has(hash []byte) bool { h := xxhash.Sum64(hash) return ((*blobloom.Filter)(f)).Has(h) } func newBF(capacity int, fpr float64) *bloomFilter { f := blobloom.NewOptimized(blobloom.Config{ Capacity: uint64(capacity), FPRate: fpr, }) return (*bloomFilter)(f) } blobloom-0.7.1/benchmarks/boom.go000066400000000000000000000030361413725631400167340ustar00rootroot00000000000000// Copyright 2020 the Blobloom authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // +build boom package benchmarks import "github.com/tylertreat/BoomFilters" type bloomFilter boom.BloomFilter func (f *bloomFilter) Add(hash []byte) { ((*boom.BloomFilter)(f)).Add(hash) } func (f *bloomFilter) Has(hash []byte) bool { return ((*boom.BloomFilter)(f)).Test(hash) } func newBF(capacity int, fpr float64) *bloomFilter { f := boom.NewBloomFilter(uint(capacity), fpr) f.SetHash(&nopHash{}) return (*bloomFilter)(f) } // No-op hash function. Assumes all data is written to it in one Write call. type nopHash struct{ data []byte } func (h *nopHash) BlockSize() int { return 1 } func (h *nopHash) Reset() {} func (h *nopHash) Size() int { return 8 } func (h *nopHash) Sum(d []byte) []byte { return append(d, h.data...) } func (h *nopHash) Sum64() uint64 { panic("not used by BoomFilters") } func (h *nopHash) Write(p []byte) (n int, err error) { if len(p) > 8 { p = p[:8] } h.data = p return len(p), nil } blobloom-0.7.1/benchmarks/devopsfaith.go000066400000000000000000000022361413725631400203150ustar00rootroot00000000000000// Copyright 2021 the Blobloom authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // +build devopsfaith package benchmarks import ( "github.com/devopsfaith/bloomfilter" "github.com/devopsfaith/bloomfilter/bloomfilter" ) type bloomFilter baseBloomfilter.Bloomfilter func newBF(capacity int, fpr float64) *bloomFilter { f := baseBloomfilter.New(bloomfilter.Config{ N: uint(capacity), P: fpr, HashName: "default", }) return (*bloomFilter)(f) } func (f *bloomFilter) Add(hash []byte) { ((*baseBloomfilter.Bloomfilter)(f)).Add(hash) } func (f *bloomFilter) Has(hash []byte) bool { return ((*baseBloomfilter.Bloomfilter)(f)).Check(hash) } blobloom-0.7.1/benchmarks/go.mod000066400000000000000000000010141413725631400165510ustar00rootroot00000000000000module github.com/greatroar/blobloom/benchmarks go 1.14 require ( github.com/bits-and-blooms/bloom/v3 v3.0.1 github.com/cespare/xxhash/v2 v2.1.1 github.com/d4l3k/messagediff v1.2.1 // indirect github.com/devopsfaith/bloomfilter v1.4.0 github.com/greatroar/blobloom v0.6.0 github.com/ipfs/bbloom v0.0.4 github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43 github.com/zeebo/xxh3 v0.12.0 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e // indirect ) replace github.com/greatroar/blobloom => ../ blobloom-0.7.1/benchmarks/go.sum000066400000000000000000001111721413725631400166050ustar00rootroot00000000000000cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/Azure/azure-sdk-for-go v16.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-autorest v10.7.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v10.15.3+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Microsoft/go-winio v0.4.3/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/NYTimes/gziphandler v1.0.1/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/abdullin/seq v0.0.0-20160510034733-d5467c17e7af/go.mod h1:5Jv4cbFiHJMsVxt52+i0Ha45fjshj6wxYr1r19tB9bw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= github.com/armon/go-metrics v0.3.4/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.15.24/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bloom/v3 v3.0.1 h1:Inlf0YXbgehxVjMPmCGv86iMCKMGPPrPSHtBF5yRHwA= github.com/bits-and-blooms/bloom/v3 v3.0.1/go.mod h1:MC8muvBzzPOFsrcdND/A7kU7kMhkqb9KI70JlZCP+C8= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coredns/coredns v1.1.2/go.mod h1:zASH/MVDgR6XZTbxvOnsZfffS+31vg6Ackf/wo1+AM0= github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U= github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denverdino/aliyungo v0.0.0-20170926055100-d3308649c661/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= github.com/devopsfaith/bloomfilter v1.4.0 h1:h8UVgr0sWxwSciqAZKd4LFugljZk8a/+DW2omzVgB08= github.com/devopsfaith/bloomfilter v1.4.0/go.mod h1:LQuMjwtBekCk8qdTHOJ7wZpEYB3J6zQEwwCPABbWMLY= github.com/devopsfaith/flatmap v0.0.0-20200601181759-8521186182fc/go.mod h1:J9Y/58s7wx7HbHT3i4UKNwLGuBB9qCf0/JUdEFGDPmA= github.com/devopsfaith/krakend-consul v1.4.0/go.mod h1:76v8AByTEzlBbiGWEzHFvT4g9BOtr8fpotYIMoac64Q= github.com/devopsfaith/krakend-gologging v1.4.0/go.mod h1:0IBy8rXN5ck5nHp5DRxOki3nPVm3Akta4X78qeNATwA= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/digitalocean/godo v1.1.1/go.mod h1:h6faOIcZ8lWIwNQ+DN7b3CgX4Kwby5T+nbpNqkUIozU= github.com/digitalocean/godo v1.10.0/go.mod h1:h6faOIcZ8lWIwNQ+DN7b3CgX4Kwby5T+nbpNqkUIozU= github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0= github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/elazarl/go-bindata-assetfs v0.0.0-20160803192304-e1a2a7ec64b0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/envoyproxy/go-control-plane v0.8.0/go.mod h1:GSSbY9P1neVhdY7G4wu+IK1rk/dqhiCC/4ExuWJZVuk= github.com/envoyproxy/protoc-gen-validate v0.0.14/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-gonic/gin v1.1.5-0.20170702092826-d459835d2b07/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= github.com/go-chi/chi v4.0.1+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-contrib/uuid v1.2.0/go.mod h1:R9zf5oXjEfersQve5ceWY37X8JR3qtDTU2WSVxbWXGE= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gophercloud/gophercloud v0.0.0-20180828235145-f29afc2cceca/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4= github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/hashicorp/consul v1.6.10/go.mod h1:rQ8araXgdbaX5K/ZoZHDpVSIX+jF0o0Zm7+A9egVur4= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-bexpr v0.1.2/go.mod h1:ANbpTX1oAql27TZkKVeW8p1w8NTdnyzPe/0qqPCKohU= github.com/hashicorp/go-checkpoint v0.0.0-20171009173528-1545e56e46de/go.mod h1:xIwEieBHERyEvaeKF/TcHh1Hu+lxPM+n2vT1+g9I4m4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-connlimit v0.1.0/go.mod h1:OUj9FGL1tPIhl/2RCfzYHrIiWj+VVPGNyVPnUX8AqS0= github.com/hashicorp/go-discover v0.0.0-20190403160810-22221edb15cd/go.mod h1:ueUgD9BeIocT7QNuvxSyJyPAM9dfifBcaWmeybb67OY= github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-memdb v0.0.0-20180223233045-1289e7fffe71/go.mod h1:kbfItVoBJwCfKXDXN4YoAXjxcFVZ7MRrJzyTX6H4giE= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= github.com/hashicorp/go-raftchunking v0.6.1/go.mod h1:cGlg3JtDy7qy6c/3Bu660Mic1JF+7lWqIwCFSb08fX0= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hil v0.0.0-20160711231837-1e86c6b523c5/go.mod h1:KHvg/R2/dPtaePb16oW4qIyzkMxXOL38xjRN64adsts= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/memberlist v0.1.5/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69/go.mod h1:/z+jUGRBlwVpUZfjute9jWaF6/HuhjuFQuL1YXzVD1Q= github.com/hashicorp/raft v1.1.1/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/serf v0.8.6/go.mod h1:P/AVgr4UHsUYqVHG1y9eFhz8S35pqhGhLZaDpfGKIMo= github.com/hashicorp/vault/api v1.0.4/go.mod h1:gDcqh3WGcR1cpF5AJz/B1UFheUEneMoIospckxBxk6Q= github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M= github.com/hashicorp/vic v1.5.1-0.20190403131502-bbfe86ec9443/go.mod h1:bEpDU35nTu0ey1EXjwNwPjI9xErAsoOCmcMb9GKvyxo= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/jarcoal/httpmock v0.0.0-20180424175123-9c70cfe4a1da/go.mod h1:ks+b9deReOc7jgqp+e7LuFiCBH6Rm5hL32cLcEAArb4= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/joyent/triton-go v0.0.0-20180628001255-830d2b111e62/go.mod h1:U+RSyWxWd04xTqnuOQxnai7XGS2PrPY2cfGoDKtMHjA= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/letgoapp/krakend-consul v0.0.0-20180406153423-b4d135ce6994/go.mod h1:cFcys9MH7oD42Dw7NVy/971Sn4zuCmtE/XjwVgk3Srw= github.com/luraproject/lura v1.4.0/go.mod h1:KIo1/+nsRZVxIO04Hkbth0GXSSzypvkFpF5KaIoLvlo= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nicolai86/scaleway-sdk v1.10.2-0.20180628010248-798f60e20bb2/go.mod h1:TLb2Sg7HQcgGdloNxkrmtgDNR9uVYF3lfdFIN4Ro6Sk= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/packethost/packngo v0.1.1-0.20180711074735-b9cb5096f54c/go.mod h1:otzZQXgoO96RTzDB/Hycg0qZcXZsWJGJRSXbmEIJ+4M= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/renier/xmlrpc v0.0.0-20170708154548-ce4a1a486c03/go.mod h1:gRAiPF5C5Nd0eyyRdqIu9qTiFSoZzpTq727b5B8fkkU= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shirou/gopsutil v0.0.0-20181107111621-48177ef5f880/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180820201707-7c9eb446e3cf/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= github.com/softlayer/softlayer-go v0.0.0-20180806151055-260589d94c7d/go.mod h1:Cw4GTlQccdRGSEf6KiMju767x0NEHE0YIVPJSaXjlsw= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tent/http-link-go v0.0.0-20130702225549-ac974c61c2f9/go.mod h1:RHkNRtSLfOK7qBTHaeSX1D6BNpI3qw7NTxsmNr4RvN8= github.com/tmthrgd/asm v0.0.0-20180106020940-4be3ab5ca701/go.mod h1:eH36fdzMoEurHdB3rmh4o3AGc67+EDT2TGaYzwdqTe4= github.com/tmthrgd/atomics v0.0.0-20180217065130-6910de195248 h1:JUDytrg3F4kx+oWd8e1Up4EHXjqgpjDoaNkCNGYZPW0= github.com/tmthrgd/atomics v0.0.0-20180217065130-6910de195248/go.mod h1:J2+dTgaX/1g3PkyL6sLBglBWfaLmAp5bQbRhSfKw9XI= github.com/tmthrgd/go-bitset v0.0.0-20180828125936-62ad9ed7ff29 h1:Bg9ER/PR7x7NcvNXTz+1r3rHO0gk4RqT/FB5KcQ2jUs= github.com/tmthrgd/go-bitset v0.0.0-20180828125936-62ad9ed7ff29/go.mod h1:SooM96OIpihI7iMZhVGbpiiO9Qevqv8vXxHlwNtefd4= github.com/tmthrgd/go-bitwise v0.0.0-20170218093117-01bef038b6bd h1:PKg5Bp8mVwropvfkNpvR1niDy+SSmH1Tg+UXMBltit0= github.com/tmthrgd/go-bitwise v0.0.0-20170218093117-01bef038b6bd/go.mod h1:Ba4ek/h+sJUzTQ03ZGD1r0lazhxd7CBoEQzFk/icxxU= github.com/tmthrgd/go-byte-test v0.0.0-20170223110042-2eb5216b83f7 h1:6KysdG7j6BH714qbpKiowHYZAgnAZqNKIWF/wNYU9Ls= github.com/tmthrgd/go-byte-test v0.0.0-20170223110042-2eb5216b83f7/go.mod h1:MEz1Lt0fxSL/ZgE7VN3yUJV0sP5I5aYecYLd9y/viEs= github.com/tmthrgd/go-hex v0.0.0-20180828131331-d1fb3dbb16a1 h1:RjxnUA5glqD1hvBTCIdDFl+IfdHlRqlXSoz6qpcY+og= github.com/tmthrgd/go-hex v0.0.0-20180828131331-d1fb3dbb16a1/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= github.com/tmthrgd/go-memset v0.0.0-20180828131805-6f4e59bf1e1d h1:gj9wtJOyhdRQ5eDC+5Ljd//pDB6gkMC7RVknqEkh8Mw= github.com/tmthrgd/go-memset v0.0.0-20180828131805-6f4e59bf1e1d/go.mod h1:xUkvcKF3VBDKFmmqCtW333lognWBHzSScj4fgjVB0Ek= github.com/tmthrgd/go-popcount v0.0.0-20180111143836-3918361d3e97 h1:dOZJr7CPLi1ud2U6BZWk+vtUFFzMPbKcZw6t1WxreDM= github.com/tmthrgd/go-popcount v0.0.0-20180111143836-3918361d3e97/go.mod h1:FcUQfrsAsSSqM3n9xf4EtPzB8tWzt58/y0AV+wNNM8Q= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43 h1:QEePdg0ty2r0t1+qwfZmQ4OOl/MB2UXIeJSpIZv56lg= github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43/go.mod h1:OYRfF6eb5wY9VRFkXJH8FFBi3plw2v+giaIu7P054pM= github.com/ugorji/go v0.0.0-20180112141927-9831f2c3ac10/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= github.com/urfave/negroni v0.3.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= github.com/vmware/govmomi v0.18.0/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= github.com/zeebo/xxh3 v0.12.0 h1:AGJ7bBsKfpYmaoTEx714qGOTjUxRo3nj5NYqUUQBrTA= github.com/zeebo/xxh3 v0.12.0/go.mod h1:AQY73TOrhF3jNsdiM9zZOb8MThrYbZONHj7ryDBaLpg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191106202628-ed6320f186d4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20170807180024-9a379c6b3e95/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190508220229-2d0786266e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200727154430-2d971f7391a4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.0.0-20180829000535-087779f1d2c9/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= istio.io/gogo-genproto v0.0.0-20190124151557-6d926a6e6feb/go.mod h1:eIDJ6jNk/IeJz6ODSksHl5Aiczy5JUq6vFhJWI5OtiI= k8s.io/api v0.0.0-20180806132203-61b11ee65332/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/api v0.0.0-20190325185214-7544f9db76f6/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/apimachinery v0.0.0-20180821005732-488889b0007f/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= k8s.io/apimachinery v0.0.0-20190223001710-c182ff3b9841/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= k8s.io/client-go v8.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= blobloom-0.7.1/benchmarks/sync.go000066400000000000000000000022341413725631400167530ustar00rootroot00000000000000// Copyright 2021 the Blobloom authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // +build sync package benchmarks import ( "encoding/binary" "github.com/greatroar/blobloom" ) type bloomFilter blobloom.SyncFilter func (f *bloomFilter) Add(hash []byte) { h := binary.BigEndian.Uint64(hash[:8]) ((*blobloom.SyncFilter)(f)).Add(h) } func (f *bloomFilter) Has(hash []byte) bool { h := binary.BigEndian.Uint64(hash[:8]) return ((*blobloom.SyncFilter)(f)).Has(h) } func newBF(capacity int, fpr float64) *bloomFilter { f := blobloom.NewSync(blobloom.Optimize(blobloom.Config{ Capacity: uint64(capacity), FPRate: fpr, })) return (*bloomFilter)(f) } blobloom-0.7.1/benchmarks/willf.go000066400000000000000000000017771413725631400171270ustar00rootroot00000000000000// Copyright 2020 the Blobloom authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // +build willf package benchmarks import "github.com/bits-and-blooms/bloom/v3" type bloomFilter bloom.BloomFilter func (f *bloomFilter) Add(hash []byte) { ((*bloom.BloomFilter)(f)).Add(hash) } func (f *bloomFilter) Has(hash []byte) bool { return ((*bloom.BloomFilter)(f)).Test(hash) } func newBF(capacity int, fpr float64) *bloomFilter { f := bloom.NewWithEstimates(uint(capacity), fpr) return (*bloomFilter)(f) } blobloom-0.7.1/bloomfilter.go000066400000000000000000000172141413725631400162040ustar00rootroot00000000000000// Copyright 2020-2021 the Blobloom authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package blobloom implements blocked Bloom filters. // // Blocked Bloom filters are an approximate set data structure: if a key has // been added to a filter, a lookup of that key returns true, but if the key // has not been added, there is a non-zero probability that the lookup still // returns true (a false positive). False negatives are impossible: if the // lookup for a key returns false, that key has not been added. // // In this package, keys are represented exclusively as hashes. Client code // is responsible for supplying a 64-bit hash value. // // Compared to standard Bloom filters, blocked Bloom filters use the CPU // cache more efficiently. A blocked Bloom filter is an array of ordinary // Bloom filters of fixed size BlockBits (the blocks). The lower half of the // hash selects the block to use. // // To achieve the same false positive rate (FPR) as a standard Bloom filter, // a blocked Bloom filter requires more memory. For an FPR of at most 2e-6 // (two in a million), it uses ~20% more memory. At 1e-10, the space required // is double that of standard Bloom filter. // // For more details, see the 2010 paper by Putze, Sanders and Singler, // https://algo2.iti.kit.edu/documents/cacheefficientbloomfilters-jea.pdf. package blobloom import "math" // BlockBits is the number of bits per block and the minimum number of bits // in a Filter. // // The value of this constant is chosen to match the L1 cache line size // of popular architectures (386, amd64, arm64). const BlockBits = 512 // MaxBits is the maximum number of bits supported by a Filter. const MaxBits = BlockBits << 32 // 256GiB. // A Filter is a blocked Bloom filter. type Filter struct { b []block // Shards. k int // Number of hash functions required. } // New constructs a Bloom filter with given numbers of bits and hash functions. // // The number of bits should be at least BlockBits; smaller values are silently // increased. // // The number of hashes reflects the number of hashes synthesized from the // single hash passed in by the client. It is silently increased to two if // a lower value is given. func New(nbits uint64, nhashes int) *Filter { if nbits < 1 { nbits = BlockBits } if nhashes < 2 { nhashes = 2 } if nbits > MaxBits { panic("nbits exceeds MaxBits") } // Round nbits up to a multiple of BlockBits. if nbits%BlockBits != 0 { nbits += BlockBits - nbits%BlockBits } return &Filter{ b: make([]block, nbits/BlockBits), k: nhashes, } } // Add insert a key with hash value h into f. func (f *Filter) Add(h uint64) { h1, h2 := uint32(h>>32), uint32(h) b := f.getblock(h2) for i := 1; i < f.k; i++ { h1, h2 = doublehash(h1, h2, i) b.setbit(h1) } } // log(1 - 1/BlockBits) computed with 128 bits precision. // Note that this is extremely close to -1/BlockBits, // which is what Wikipedia would have us use: // https://en.wikipedia.org/wiki/Bloom_filter#Approximating_the_number_of_items_in_a_Bloom_filter. const log1minus1divBlockbits = -0.0019550348358033505576274922418668121377 // Cardinality estimates the number of distinct keys added to f. // // The estimate is most reliable when f is filled to roughly its capacity. // It gets worse as f gets more densely filled. When one of the blocks is // entirely filled, the estimate becomes +Inf. // // The return value is the maximum likelihood estimate of Papapetrou, Siberski // and Nejdl, summed over the blocks // (https://www.win.tue.nl/~opapapetrou/papers/Bloomfilters-DAPD.pdf). func (f *Filter) Cardinality() float64 { return f.cardinality(onescount) } func (f *Filter) cardinality(onescount func(*block) int) float64 { k := float64(f.k) - 1 // The probability of some bit not being set in a single insertion is // p0 = (1-1/BlockBits)^k. // // logProb0Inv = 1 / log(p0) = 1 / (k*log(1-1/BlockBits)). logProb0Inv := 1 / (k * log1minus1divBlockbits) var n float64 for i := range f.b { ones := onescount(&f.b[i]) if ones == 0 { continue } n += math.Log1p(-float64(ones) / BlockBits) } return n * logProb0Inv } // Clear resets f to its empty state. func (f *Filter) Clear() { for i := 0; i < len(f.b); i++ { f.b[i] = block{} } } // Empty reports whether f contains no keys. func (f *Filter) Empty() bool { for i := 0; i < len(f.b); i++ { if f.b[i] != (block{}) { return false } } return true } // Fill set f to a completely full filter. // After Fill, Has returns true for any key. func (f *Filter) Fill() { for i := 0; i < len(f.b); i++ { for j := 0; j < blockWords; j++ { f.b[i][j] = ^uint32(0) } } } // Has reports whether a key with hash value h has been added. // It may return a false positive. func (f *Filter) Has(h uint64) bool { h1, h2 := uint32(h>>32), uint32(h) b := f.getblock(h2) for i := 1; i < f.k; i++ { h1, h2 = doublehash(h1, h2, i) if !b.getbit(h1) { return false } } return true } // doublehash generates the hash values to use in iteration i of // enhanced double hashing from the values h1, h2 of the previous iteration. // See https://www.ccs.neu.edu/home/pete/pub/bloom-filters-verification.pdf. func doublehash(h1, h2 uint32, i int) (uint32, uint32) { h1 = h1 + h2 h2 = h2 + uint32(i) return h1, h2 } // NumBits returns the number of bits of f. func (f *Filter) NumBits() uint64 { return BlockBits * uint64(len(f.b)) } func checkBinop(f, g *Filter) { if len(f.b) != len(g.b) { panic("Bloom filters do not have the same number of bits") } if f.k != g.k { panic("Bloom filters do not have the same number of hash functions") } } // Intersect sets f to the intersection of f and g. // // Intersect panics when f and g do not have the same number of bits and // hash functions. Both Filters must be using the same hash function(s), // but Intersect cannot check this. // // Since Bloom filters may return false positives, Has may return true for // a key that was not in both f and g. // // After Intersect, the estimates from f.Cardinality and f.FPRate should be // considered unreliable. func (f *Filter) Intersect(g *Filter) { checkBinop(f, g) f.intersect(g) } // Union sets f to the union of f and g. // // Union panics when f and g do not have the same number of bits and // hash functions. Both Filters must be using the same hash function(s), // but Union cannot check this. func (f *Filter) Union(g *Filter) { checkBinop(f, g) f.union(g) } const ( wordSize = 32 blockWords = BlockBits / wordSize ) // A block is a fixed-size Bloom filter, used as a shard of a Filter. type block [blockWords]uint32 func (f *Filter) getblock(h2 uint32) *block { i := reducerange(h2, uint32(len(f.b))) return &f.b[i] } // reducerange maps i to an integer in the range [0,n). // https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ func reducerange(i, n uint32) uint32 { return uint32((uint64(i) * uint64(n)) >> 32) } // getbit reports whether bit (i modulo BlockBits) is set. func (b *block) getbit(i uint32) bool { bit := uint32(1) << (i % wordSize) x := (*b)[(i/wordSize)%blockWords] & bit return x != 0 } // setbit sets bit (i modulo BlockBits) of b. func (b *block) setbit(i uint32) { bit := uint32(1) << (i % wordSize) (*b)[(i/wordSize)%blockWords] |= bit } blobloom-0.7.1/bloomfilter_test.go000066400000000000000000000147061413725631400172460ustar00rootroot00000000000000// Copyright 2020 the Blobloom authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package blobloom import ( "crypto/sha256" "encoding/binary" "encoding/hex" "math" "math/rand" "testing" "github.com/stretchr/testify/assert" ) func TestSimple(t *testing.T) { t.Parallel() keys := randomU64(10000, 0x758e326) for _, config := range []struct { nbits uint64 nhashes int }{ {1, 2}, {1024, 4}, {100, 3}, {10000, 7}, {1000000, 14}, } { f := New(config.nbits, config.nhashes) assert.GreaterOrEqual(t, f.NumBits(), config.nbits) assert.LessOrEqual(t, f.NumBits(), config.nbits+BlockBits) assert.True(t, f.Empty()) for _, k := range keys { assert.False(t, f.Has(k)) } for _, k := range keys { f.Add(k) } assert.False(t, f.Empty()) for _, k := range keys { assert.True(t, f.Has(k)) } f.Clear() assert.True(t, f.Empty()) for _, k := range keys { assert.False(t, f.Has(k)) } f.Fill() assert.False(t, f.Empty()) for _, k := range keys { assert.True(t, f.Has(k)) } } } func TestUse(t *testing.T) { t.Parallel() const n = 100000 // For FPR = .01, n = 100000, the optimal number of bits is 958505.84 // for a standard Bloom filter. f := NewOptimized(Config{ Capacity: n, FPRate: .01, }) if f.NumBits() < 958506 { t.Fatalf("bloom filter with %d bits too small", f.NumBits()) } t.Logf("k = %d; m/n = %d/%d = %.3f", f.k, f.NumBits(), n, float64(f.NumBits())/n) // Generate random hash values for n keys. Pretend the keys are all distinct, // even if the hashes are not. r := rand.New(rand.NewSource(0xb1007)) hashes := make([]uint64, n) for i := range hashes { hashes[i] = r.Uint64() } for _, h := range hashes { f.Add(h) } for _, h := range hashes { if !f.Has(h) { t.Errorf("%032x added to Bloom filter but not found", h) } } // Generate some more random hashes to get a sense of the FPR. // Pretend these represent unique keys, distinct from the ones we added. const nTest = 10000 fp := 0 for i := 0; i < nTest; i++ { if f.Has(r.Uint64()) { fp++ } } fpr := float64(fp) / nTest assert.Less(t, fpr, .02) t.Logf("FPR = %.5f\n", fpr) } // Test robustness against 32-bit hash functions. func TestHash32(t *testing.T) { t.Parallel() const n = 400 f := NewOptimized(Config{ Capacity: n, FPRate: .01, }) r := rand.New(rand.NewSource(32)) for i := 0; i < n; i++ { f.Add(uint64(r.Uint32())) } const nrounds = 8 fp := 0 for i := n; i < nrounds*n; i++ { if f.Has(uint64(r.Uint32())) { fp++ } } fprate := float64(fp) / (nrounds * n) t.Logf("FP rate = %.2f%%", 100*fprate) assert.LessOrEqual(t, fprate, .1) } func TestDoubleHashing(t *testing.T) { t.Parallel() var h1, h2 uint32 = 0, 0 for i := 0; i < 20; i++ { h1, h2 = doublehash(h1, h2, i) assert.NotEqual(t, h2, 0) } } func TestReducerange(t *testing.T) { t.Parallel() for i := 0; i < 40000; i++ { m := rand.Uint32() j := reducerange(rand.Uint32(), m) if m == 0 { assert.Equal(t, j, 0) } assert.Less(t, j, m) } } func TestCardinality(t *testing.T) { t.Parallel() const cap = 1e4 f := NewOptimized(Config{ Capacity: cap, FPRate: .0015, }) assert.EqualValues(t, 0, f.Cardinality()) r := rand.New(rand.NewSource(0x81feae2b)) var sumN, sumNhat float64 for n := 1.0; n <= 5*cap; n++ { f.Add(r.Uint64()) nhat := f.Cardinality() assert.InDelta(t, 1, nhat/float64(n), 0.09) sumN += n sumNhat += nhat if int(n)%cap == 0 { // On average, we want to be less than a percent off. assert.InDelta(t, 1, sumNhat/sumN, 0.008) } } } func TestCardinalityFull(t *testing.T) { t.Parallel() f := New(BlockBits, 2) for i := range f.b { for j := range f.b[i] { f.b[i][j] = ^uint32(0) } } assert.Equal(t, math.Inf(+1), f.Cardinality()) } func TestIntersect(t *testing.T) { t.Parallel() const n uint64 = 1e4 const seed = 0x5544332211 hashes := randomU64(int(n), seed) f := NewOptimized(Config{Capacity: n, FPRate: 1e-3}) g := NewOptimized(Config{Capacity: n, FPRate: 1e-3}) i := NewOptimized(Config{Capacity: n, FPRate: 1e-3}) for _, h := range hashes[:n/3] { f.Add(h) } for _, h := range hashes[n/3 : 2*n/3] { f.Add(h) g.Add(h) i.Add(h) } for _, h := range hashes[n/3:] { g.Add(h) } expectFPR := math.Min(f.FPRate(n), g.FPRate(n)) f.Intersect(g) assert.NotEqual(t, i, g) for _, h := range hashes[n/3 : 2*n/3] { assert.True(t, f.Has(h)) } var fp uint64 for _, h := range hashes { if f.Has(h) && !i.Has(h) { fp++ } } actualFPR := float64(fp) / float64(n) assert.Less(t, actualFPR, 2*expectFPR) t.Logf("FPR = %f", actualFPR) assert.Panics(t, func() { f.Intersect(New(f.NumBits(), 9)) }) assert.Panics(t, func() { f.Union(New(n+BlockBits, f.k)) }) } func TestUnion(t *testing.T) { t.Parallel() const n = 1e5 hashes := randomU64(n, 0xa6e98fb) f := New(n, 5) g := New(n, 5) u := New(n, 5) for _, h := range hashes[:n/2] { f.Add(h) u.Add(h) } for _, h := range hashes[n/2:] { g.Add(h) u.Add(h) } assert.NotEqual(t, f, g) f.Union(g) assert.Equal(t, u, f) assert.NotEqual(t, u, g) g.Union(f) assert.Equal(t, u, g) assert.Panics(t, func() { f.Union(New(n, 4)) }) assert.Panics(t, func() { f.Union(New(n+BlockBits, 5)) }) } func randomU64(n int, seed int64) []uint64 { r := rand.New(rand.NewSource(seed)) p := make([]uint64, n) for i := range p { p[i] = r.Uint64() } return p } func TestUnionSmall(t *testing.T) { t.Parallel() f := New(BlockBits, 2) g := New(BlockBits, 2) g.Add(42) f.Union(g) assert.True(t, f.Has(42)) } // This test ensures that the switch from 64-bit to 32-bit words did not // alter the little-endian serialization of blocks. func TestBlockLayout(t *testing.T) { t.Parallel() var b block b.setbit(0) b.setbit(1) b.setbit(111) b.setbit(499) assert.Equal(t, BlockBits, 8*binary.Size(b)) h := sha256.New() binary.Write(h, binary.LittleEndian, b) expect := "aa7f8c411600fa387f0c10641eab428a7ed2f27a86171ac69f0e2087b2aa9140" assert.Equal(t, expect, hex.EncodeToString(h.Sum(nil))) } blobloom-0.7.1/example_test.go000066400000000000000000000131031413725631400163510ustar00rootroot00000000000000// Copyright 2020 the Blobloom authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package blobloom_test import ( "crypto/sha256" "encoding/binary" "fmt" "hash/fnv" "io" "math" "sync" "github.com/greatroar/blobloom" ) func Example_fnv() { // This example uses the hash/fnv package from the standard Go library. f := blobloom.New(10000, 5) h := fnv.New64() messages := []string{ "Hello!", "Welcome!", "Mind your step!", "Have fun!", "Goodbye!", } for _, msg := range messages { h.Reset() io.WriteString(h, msg) f.Add(h.Sum64()) } for _, msg := range messages { h.Reset() io.WriteString(h, msg) if f.Has(h.Sum64()) { fmt.Println(msg) } else { panic("Bloom filter didn't get the message") } } // Output: // Hello! // Welcome! // Mind your step! // Have fun! // Goodbye! } func Example_sha224() { // If you have items addressed by a cryptographic hash, // you can use a prefix of it as the hash value for a Bloom filter. // // If the cryptohashes denote objects from an untrusted source, // the Bloom filter can be tricked into giving false positives for // chosen objects, because it only uses a small part of the hash // that can easily be broken (by a birthday attack). If that can // cause problems in your application, first run SipHash on the // full cryptohash to get the hash value for the Bloom filter: // // import "github.com/dchest/siphash" // h := siphash.Hash(secret1, secret2, key[:]) // A list of files, identified by their SHA-224. files := []string{ "\x85\x52\xd8\xb7\xa7\xdc\x54\x76\xcb\x9e\x25\xde\xe6\x9a\x80\x91\x29\x07\x64\xb7\xf2\xa6\x4f\xe6\xe7\x8e\x95\x68", "\xa0\xad\x8f\x63\x90\x72\x74\x7b\xc3\x43\x09\x45\x94\x0e\x7c\x73\xb8\x34\x93\xf1\x77\x90\x0f\xd2\x7d\x09\x65\x94", "\x7b\xd3\xdb\x48\x1e\x7b\x05\x2c\x88\x18\x68\xcc\x13\xc3\x04\x34\x43\x2d\x7b\x49\x24\x74\x70\x33\xd2\xe8\x6e\x73", } // first64 extracts the first 64 bits of a key as a uint64. // The choice of big vs. little-endian is arbitrary. first64 := func(key []byte) uint64 { return binary.BigEndian.Uint64(key[:8]) } f := blobloom.NewOptimized(blobloom.Config{Capacity: 600, FPRate: .002}) for _, filehash := range files { f.Add(first64([]byte(filehash))) } for _, s := range []string{"Hello, world!", "Goodbye"} { h := sha256.Sum224([]byte(s)) found := f.Has(first64(h[:])) if found { fmt.Printf("Found: %v\n", s) } } // Output: // Found: Hello, world! } func ExampleOptimize() { cfg := blobloom.Config{ // We want to insert a billion keys and get a false positive rate of // one in a million, but we only have 2GiB (= 2^31 bytes) to spare. Capacity: 1e9, FPRate: 1e-6, MaxBits: 8 * 1 << 31, } nbits, nhashes := blobloom.Optimize(cfg) fpr := blobloom.FPRate(cfg.Capacity, nbits, nhashes) // How big will the filter be and what FP rate will we achieve? fmt.Printf("size = %dMiB\nfpr = %.3f\n", nbits/(8<<20), fpr) // Output: // size = 2048MiB // fpr = 0.001 } var hashes [200]uint64 func init() { for i := range hashes { hashes[i] = uint64(i) } } func ExampleFilter_Cardinality_infinity() { // To handle the case of Cardinality returning +Inf, track the number of // calls to Add and compute the minimum. // This Bloom filter is constructed with too many hash functions // to force +Inf. f := blobloom.New(512, 100) var numAdded int add := func(h uint64) { f.Add(h) numAdded++ } for _, h := range hashes { add(h) } estimate := f.Cardinality() fmt.Printf("blobloom's estimate: %.2f\n", estimate) fmt.Printf("number of calls to Add: %d\n", numAdded) estimate = math.Min(estimate, float64(numAdded)) fmt.Printf("combined estimate: %.2f\n", estimate) // Output: // blobloom's estimate: +Inf // number of calls to Add: 200 // combined estimate: 200.00 } const nworkers = 4 func getKeys(keys chan<- string) { keys <- "hello" keys <- "goodbye" close(keys) } func hash(key string) uint64 { h := fnv.New64() io.WriteString(h, key) return h.Sum64() } func ExampleFilter_Union() { // Union can be used to fill a Bloom filter using multiple goroutines. // // Each goroutine allocates a filter, so the memory use increases // by a factor nworkers-1 compared to a sequential version // or a SyncFilter. keys := make(chan string, nworkers) filters := make(chan *blobloom.Filter, nworkers) go getKeys(keys) for i := 0; i < nworkers; i++ { go func() { f := blobloom.New(1<<20, 6) for key := range keys { f.Add(hash(key)) } filters <- f }() } f := <-filters for i := 1; i < nworkers; i++ { f.Union(<-filters) } // Output: } func ExampleSyncFilter() { // Multiple goroutines can Add to a SyncFilter concurrently, // without requiring separate synchronization. f := blobloom.NewSync(1<<20, 6) var wg sync.WaitGroup add := func(hs []uint64) { for _, h := range hs { f.Add(h) } wg.Done() } wg.Add(2) half := len(hashes) / 2 go add(hashes[:half]) go add(hashes[half:]) wg.Wait() // Wait for updating goroutines to complete. for _, h := range hashes { if !f.Has(h) { fmt.Printf("hash %d added but not retrieved\n", h) } } // Output: } blobloom-0.7.1/examples/000077500000000000000000000000001413725631400151505ustar00rootroot00000000000000blobloom-0.7.1/examples/bloomstat/000077500000000000000000000000001413725631400171545ustar00rootroot00000000000000blobloom-0.7.1/examples/bloomstat/main.go000066400000000000000000000053021413725631400204270ustar00rootroot00000000000000// Copyright 2021 the Blobloom authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Bloomstat is a utility for estimating Bloom filter sizes. package main import ( "fmt" "io" "log" "os" "strconv" "strings" "github.com/greatroar/blobloom" ) const usage = `usage: bloomstat capacity false-positive-rate [max-memory] The maximum memory may be specified as "10MB", "1.5GiB", etc.` func main() { if len(os.Args) < 2 { fmt.Fprintln(os.Stderr, usage) os.Exit(1) } var ( capacity = parse("capacity", os.Args[1]) fpr = parse("false positive rate", os.Args[2]) maxsize float64 ) if len(os.Args) > 3 { maxsize = parseMem(os.Args[3]) } bits, hashes := blobloom.Optimize(blobloom.Config{ Capacity: uint64(capacity), FPRate: fpr, MaxBits: uint64(8 * maxsize), }) size, unit := memsize(float64(bits)) bitsPerKey := float64(bits) / capacity expectedFpr := blobloom.FPRate(uint64(capacity), bits, hashes) fmt.Printf("%d bits, %.02f %s\n"+ "%.02f bits/%.02f B per key\n"+ "%d hashes\n"+ "%.04f expected false positive rate\n", bits, size, unit, bitsPerKey, bitsPerKey/8, hashes, expectedFpr) } const ( kiB = 1 << 10 MiB = 1 << 20 GiB = 1 << 30 ) func memsize(bits float64) (size float64, unit string) { size = float64(bits) / 8 switch { case size >= GiB: size /= GiB unit = "GiB" case size >= MiB: size /= MiB unit = "MiB" case size >= kiB: size /= kiB unit = "kiB" default: unit = "B" } return } func parse(name, num string) float64 { v, err := strconv.ParseFloat(num, 64) switch e := err.(type) { case nil: case *strconv.NumError: log.Fatalf("%s %q: %v", name, e.Num, e.Err) default: log.Fatalf("%s: %v", name, err) } if v < 0 { log.Fatalf("%s must be >= 0", name) } return v } func parseMem(s string) float64 { var ( size float64 unit string ) n, err := fmt.Sscanf(s, "%f%s", &size, &unit) switch err { case nil: case io.EOF: if n == 1 { // Default to bytes. unit = "b" } else { log.Fatal("max memory: invalid input") } default: log.Fatal("max memory:", err) } switch strings.ToLower(unit) { case "kb", "kib": size *= kiB case "mb", "mib": size *= MiB case "gb", "gib": size *= GiB } return size } blobloom-0.7.1/examples/spellcheck/000077500000000000000000000000001413725631400172655ustar00rootroot00000000000000blobloom-0.7.1/examples/spellcheck/main.go000066400000000000000000000052411413725631400205420ustar00rootroot00000000000000// Copyright 2020-2021 the Blobloom authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // This package implements a toy interactive spell checker. // // It reads a dictionary from /usr/share/dict/words and a text from standard // input. It then reports any misspelled words on standard output. package main import ( "bufio" "bytes" "fmt" "hash/maphash" "log" "os" "unicode" "github.com/greatroar/blobloom" ) func main() { dict := loadDictionary() sc := bufio.NewScanner(os.Stdin) sc.Split(bufio.ScanWords) for sc.Scan() { word := normalize(sc.Bytes()) if !dict.has(word) { fmt.Printf(">>> %s\n", word) } } if err := sc.Err(); err != nil { log.Fatal(err) } } // A Bloom filter with a randomized hash function. type bloomfilter struct { *blobloom.Filter maphash.Seed } func newBloomfilter(capacity uint64, fprate float64) *bloomfilter { cfg := blobloom.Config{Capacity: capacity, FPRate: .001} return &bloomfilter{ Filter: blobloom.NewOptimized(cfg), Seed: maphash.MakeSeed(), } } func (f *bloomfilter) add(key []byte) { f.Filter.Add(f.hash(key)) } func (f *bloomfilter) has(key []byte) bool { return f.Filter.Has(f.hash(key)) } func (f *bloomfilter) hash(key []byte) uint64 { var h maphash.Hash h.SetSeed(f.Seed) _, _ = h.Write(key) return h.Sum64() } func normalize(word []byte) []byte { word = bytes.TrimFunc(word, unicode.IsPunct) word = bytes.ToLower(word) return word } // To estimate the number of keys without scanning the file twice, we need // an estimate of the average length of a word. This comes close for English. const avgWordLength = 10 func loadDictionary() *bloomfilter { f, err := os.Open("/usr/share/dict/words") if err != nil { log.Fatal(err) } defer f.Close() info, err := f.Stat() if err != nil { log.Fatal(err) } filesize := uint64(info.Size()) dict := newBloomfilter(filesize/avgWordLength, .001) sc := bufio.NewScanner(f) for sc.Scan() { dict.add(normalize(sc.Bytes())) filesize-- // Subtract newline, for fairness. } if err := sc.Err(); err != nil { log.Fatal(err) } log.Printf("dictionary loaded: %dkiB on disk, %dkiB in memory", filesize/1024, dict.NumBits()/(8*1024)) return dict } blobloom-0.7.1/go.mod000066400000000000000000000001321413725631400144340ustar00rootroot00000000000000module github.com/greatroar/blobloom go 1.14 require github.com/stretchr/testify v1.6.1 blobloom-0.7.1/go.sum000066400000000000000000000020001413725631400144550ustar00rootroot00000000000000github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= blobloom-0.7.1/optimize.go000066400000000000000000000127711413725631400155310ustar00rootroot00000000000000// Copyright 2020 the Blobloom authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package blobloom import "math" // A Config holds parameters for Optimize or NewOptimized. type Config struct { // Capacity is the expected number of distinct keys to be added. // More keys can always be added, but the false positive rate can be // expected to drop below FPRate if their number exceeds the Capacity. Capacity uint64 // Desired lower bound on the false positive rate when the Bloom filter // has been filled to its capacity. FPRate must be between zero // (exclusive) and one (inclusive). FPRate float64 // Maximum size of the Bloom filter in bits. Zero means the global // MaxBits constant. A value less than BlockBits means BlockBits. MaxBits uint64 // Trigger the "contains filtered or unexported fields" message for // forward compatibility and force the caller to use named fields. _ struct{} } // NewOptimized is shorthand for New(Optimize(config)). func NewOptimized(config Config) *Filter { return New(Optimize(config)) } // NewSyncOptimized is shorthand for New(Optimize(config)). func NewSyncOptimized(config Config) *SyncFilter { return NewSync(Optimize(config)) } // Optimize returns numbers of keys and hash functions that achieve the // desired false positive described by config. // // Optimize panics when config.FPRate is invalid. // // The estimated number of bits is imprecise for false positives rates below // ca. 1e-15. func Optimize(config Config) (nbits uint64, nhashes int) { n := float64(config.Capacity) p := config.FPRate if p <= 0 || p > 1 { panic("false positive rate for a Bloom filter must be > 0, <= 1") } if n == 0 { // Assume the client wants to add at least one key; log2(0) = -inf. n = 1 } // The optimal nbits/n is c = -log2(p) / ln(2) for a vanilla Bloom filter. c := math.Ceil(-math.Log2(p) / math.Ln2) if c < float64(len(correctC)) { c = float64(correctC[int(c)]) } else { // We can't achieve the desired FPR. Just triple the number of bits. c *= 3 } nbits = uint64(c * n) // Round up to a multiple of BlockBits. if nbits%BlockBits != 0 { nbits += BlockBits - nbits%BlockBits } var maxbits uint64 = MaxBits if config.MaxBits != 0 && config.MaxBits < maxbits { maxbits = config.MaxBits if maxbits < BlockBits { maxbits = BlockBits } } if nbits > maxbits { nbits = maxbits // Round down to a multiple of BlockBits. nbits -= nbits % BlockBits } // The corresponding optimal number of hash functions is k = c * log(2). // Try rounding up and down to see which rounding is better. c = float64(nbits) / n k := c * math.Ln2 if k < 1 { nhashes = 1 return nbits, nhashes } ceilK, floorK := math.Floor(k), math.Ceil(k) if ceilK == floorK { return nbits, int(ceilK) } fprCeil, _ := fpRate(c, math.Ceil(k)) fprFloor, _ := fpRate(c, math.Floor(k)) if fprFloor < fprCeil { k = floorK } else { k = ceilK } return nbits, int(k) } // correctC maps c = m/n for a vanilla Bloom filter to the c' for a // blocked Bloom filter. // // This is Putze et al.'s Table I, extended down to zero. // For c > 34, the values become huge and are hard to compute. var correctC = []byte{ 1, 1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 20, 21, 23, 25, 26, 28, 30, 32, 35, 38, 40, 44, 48, 51, 58, 64, 74, 90, } // FPRate computes an estimate of the false positive rate of a Bloom filter // after nkeys distinct keys have been added. func FPRate(nkeys, nbits uint64, nhashes int) float64 { if nkeys == 0 { return 0 } p, _ := fpRate(float64(nbits)/float64(nkeys), float64(nhashes)) return p } func fpRate(c, k float64) (p float64, iter int) { switch { case c == 0: panic("0 bits per key is too few") case k == 0: panic("0 hashes is too few") } // Putze et al.'s Equation (3). // // The Poisson distribution has a single spike around its mean // BlockBits/c that gets slimmer and further away from zero as c tends // to zero (the Bloom filter gets more filled). We start at the mean, // then add terms left and right of it until their relative contribution // drops below ε. const ε = 1e-9 mean := BlockBits / c // Ceil to make sure we start at one, not zero. i := math.Ceil(mean) p = math.Exp(logPoisson(mean, i) + logFprBlock(BlockBits/i, k)) for j := i - 1; j > 0; j-- { add := math.Exp(logPoisson(mean, j) + logFprBlock(BlockBits/j, k)) p += add iter++ if add/p < ε { break } } for j := i + 1; ; j++ { add := math.Exp(logPoisson(mean, j) + logFprBlock(BlockBits/j, k)) p += add iter++ if add/p < ε { break } } return p, iter } // FPRate computes an estimate of f's false positive rate after nkeys distinct // keys have been added. func (f *Filter) FPRate(nkeys uint64) float64 { return FPRate(nkeys, f.NumBits(), f.k) } // Log of the FPR of a single block, FPR = (1 - exp(-k/c))^k. func logFprBlock(c, k float64) float64 { return k * math.Log1p(-math.Exp(-k/c)) } // Log of the Poisson distribution's pmf. func logPoisson(λ, k float64) float64 { lg, _ := math.Lgamma(k + 1) return k*math.Log(λ) - λ - lg } blobloom-0.7.1/optimize_test.go000066400000000000000000000074421413725631400165670ustar00rootroot00000000000000// Copyright 2020 the Blobloom authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package blobloom import ( "fmt" "math" "testing" "github.com/stretchr/testify/assert" ) func TestFPRate(t *testing.T) { t.Parallel() // FP rate is zero when no keys have been inserted. assert.EqualValues(t, 0, FPRate(0, 100, 3)) // FP rate is close to one when the capacity is greatly exceeded. nhashes := 100.0 * math.Ln2 assert.InDelta(t, 1.0, FPRate(1e9, 1e8, int(nhashes)), 1e-7) // Examples from Putze et al., page 4. // XXX We compute 0.023041, which is confirmed by PARI/GP and SciPy. // Is the rounding in the paper off? assert.InDelta(t, 0.0231, FPRate(1, 8, 5), 6e-5) // XXX This one is only accurate to one digit. // The required number does not occur in the series expansion either, // the closest partial sum being 1.9536e-4. assert.InDelta(t, 1.94e-4, FPRate(1, 20, 14), 3e-5) } func TestFPRateConvergence(t *testing.T) { for _, c := range []struct { c, k float64 iter int }{ {.01, 1, 2500}, {.1, 1, 2000}, {3, 2, 200}, {4, 2, 200}, {6, 3, 200}, {8, 5, 200}, {20, 14, 100}, {30, 20, 100}, } { t.Run(fmt.Sprintf("c=%f,k=%d", c.c, int(c.k)), func(t *testing.T) { t.Parallel() fpr, iterations := fpRate(c.c, c.k) t.Logf("fpr = %f", fpr) assert.Less(t, iterations, c.iter) }) } } func TestFPRateCorrectC(t *testing.T) { t.Parallel() // Try to reconstruct the correction table. We may be one bit off. for i, expect := range correctC[1:] { c := float64(i + 1) k := float64(c) * math.Ln2 fprBlock := math.Exp(logFprBlock(c, k)) cprime := c for { if p, _ := fpRate(cprime, k); p <= fprBlock { break } cprime++ k = cprime * math.Ln2 } assert.InDelta(t, float64(expect), cprime, 1) } } func TestFPRateInvalidInput(t *testing.T) { assert.Panics(t, func() { FPRate(10, 0, 2) }) assert.Panics(t, func() { FPRate(10, 2, 0) }) } func TestNewOptimizedMaxFPR(t *testing.T) { t.Parallel() f := NewOptimized(Config{ Capacity: 0, FPRate: 1, }) assert.EqualValues(t, BlockBits, f.NumBits()) } func TestMaxBits(t *testing.T) { t.Parallel() for _, c := range []struct { want, expect uint64 }{ {1, BlockBits}, {BlockBits - 1, BlockBits}, {BlockBits + 1, BlockBits}, {2*BlockBits - 1, BlockBits}, {4<<20 - 1, 4<<20 - BlockBits}, {4<<20 + 1, 4 << 20}, {4<<20 + BlockBits, 4<<20 + BlockBits}, } { nbits, nhashes := Optimize(Config{ // Ask for tiny FPR with a huge number of keys. Capacity: 2 * c.want, FPRate: 1e-10, MaxBits: c.want, }) // Optimize should round down to multiple of BlockBits. assert.LessOrEqual(t, nbits, c.expect) assert.EqualValues(t, 0, nbits%BlockBits) f := New(nbits, nhashes) assert.Equal(t, c.expect, f.NumBits()) } } func TestOptimizeFewBits(t *testing.T) { t.Parallel() for _, config := range []Config{ { Capacity: 1, FPRate: .99, MaxBits: 1, }, { Capacity: 100000, FPRate: 0.01, MaxBits: 408, }, } { // Optimize should give nbits >= BlockBits. nbits, nhashes := Optimize(config) assert.EqualValues(t, BlockBits, nbits) assert.Greater(t, nhashes, 0) } } func TestOptimizeInvalidFPRate(t *testing.T) { t.Parallel() assert.Panics(t, func() { Optimize(Config{FPRate: 0}) }) assert.Panics(t, func() { Optimize(Config{FPRate: 1.0000001}) }) } blobloom-0.7.1/setop_64bit.go000066400000000000000000000060411413725631400160240ustar00rootroot00000000000000// Copyright 2020 the Blobloom authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build (amd64 || arm64) && !nounsafe // +build amd64 arm64 // +build !nounsafe package blobloom import ( "math/bits" "sync/atomic" "unsafe" ) // Block reinterpreted as array of uint64. type block64 [BlockBits / 64]uint64 func (f *Filter) intersect(g *Filter) { a, b := f.b, g.b for len(a) >= 2 { p := (*block64)(unsafe.Pointer(&a[0])) q := (*block64)(unsafe.Pointer(&b[0])) p[0] &= q[0] p[1] &= q[1] p[2] &= q[2] p[3] &= q[3] p[4] &= q[4] p[5] &= q[5] p[6] &= q[6] p[7] &= q[7] p = (*block64)(unsafe.Pointer(&a[1])) q = (*block64)(unsafe.Pointer(&b[1])) p[0] &= q[0] p[1] &= q[1] p[2] &= q[2] p[3] &= q[3] p[4] &= q[4] p[5] &= q[5] p[6] &= q[6] p[7] &= q[7] a, b = a[2:], b[2:] } if len(a) > 0 { p := (*block64)(unsafe.Pointer(&a[0])) q := (*block64)(unsafe.Pointer(&b[0])) p[0] &= q[0] p[1] &= q[1] p[2] &= q[2] p[3] &= q[3] p[4] &= q[4] p[5] &= q[5] p[6] &= q[6] p[7] &= q[7] } } func (f *Filter) union(g *Filter) { a, b := f.b, g.b for len(a) >= 2 { p := (*block64)(unsafe.Pointer(&a[0])) q := (*block64)(unsafe.Pointer(&b[0])) p[0] |= q[0] p[1] |= q[1] p[2] |= q[2] p[3] |= q[3] p[4] |= q[4] p[5] |= q[5] p[6] |= q[6] p[7] |= q[7] p = (*block64)(unsafe.Pointer(&a[1])) q = (*block64)(unsafe.Pointer(&b[1])) p[0] |= q[0] p[1] |= q[1] p[2] |= q[2] p[3] |= q[3] p[4] |= q[4] p[5] |= q[5] p[6] |= q[6] p[7] |= q[7] a, b = a[2:], b[2:] } if len(a) > 0 { p := (*block64)(unsafe.Pointer(&a[0])) q := (*block64)(unsafe.Pointer(&b[0])) p[0] |= q[0] p[1] |= q[1] p[2] |= q[2] p[3] |= q[3] p[4] |= q[4] p[5] |= q[5] p[6] |= q[6] p[7] |= q[7] } } func onescount(b *block) (n int) { p := (*block64)(unsafe.Pointer(&b[0])) n += bits.OnesCount64(p[0]) n += bits.OnesCount64(p[1]) n += bits.OnesCount64(p[2]) n += bits.OnesCount64(p[3]) n += bits.OnesCount64(p[4]) n += bits.OnesCount64(p[5]) n += bits.OnesCount64(p[6]) n += bits.OnesCount64(p[7]) return n } func onescountAtomic(b *block) (n int) { p := (*block64)(unsafe.Pointer(&b[0])) n += bits.OnesCount64(atomic.LoadUint64(&p[0])) n += bits.OnesCount64(atomic.LoadUint64(&p[1])) n += bits.OnesCount64(atomic.LoadUint64(&p[2])) n += bits.OnesCount64(atomic.LoadUint64(&p[3])) n += bits.OnesCount64(atomic.LoadUint64(&p[4])) n += bits.OnesCount64(atomic.LoadUint64(&p[5])) n += bits.OnesCount64(atomic.LoadUint64(&p[6])) n += bits.OnesCount64(atomic.LoadUint64(&p[7])) return n } blobloom-0.7.1/setop_other.go000066400000000000000000000054151413725631400162210ustar00rootroot00000000000000// Copyright 2020 the Blobloom authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build (!amd64 && !arm64) || nounsafe // +build !amd64,!arm64 nounsafe package blobloom import ( "math/bits" "sync/atomic" ) func (f *Filter) intersect(g *Filter) { for i := range f.b { f.b[i].intersect(&g.b[i]) } } func (f *Filter) union(g *Filter) { for i := range f.b { f.b[i].union(&g.b[i]) } } func (b *block) intersect(c *block) { b[0] &= c[0] b[1] &= c[1] b[2] &= c[2] b[3] &= c[3] b[4] &= c[4] b[5] &= c[5] b[6] &= c[6] b[7] &= c[7] b[8] &= c[8] b[9] &= c[9] b[10] &= c[10] b[11] &= c[11] b[12] &= c[12] b[13] &= c[13] b[14] &= c[14] b[15] &= c[15] } func (b *block) union(c *block) { b[0] |= c[0] b[1] |= c[1] b[2] |= c[2] b[3] |= c[3] b[4] |= c[4] b[5] |= c[5] b[6] |= c[6] b[7] |= c[7] b[8] |= c[8] b[9] |= c[9] b[10] |= c[10] b[11] |= c[11] b[12] |= c[12] b[13] |= c[13] b[14] |= c[14] b[15] |= c[15] } func onescount(b *block) (n int) { n += bits.OnesCount32(b[0]) n += bits.OnesCount32(b[1]) n += bits.OnesCount32(b[2]) n += bits.OnesCount32(b[3]) n += bits.OnesCount32(b[4]) n += bits.OnesCount32(b[5]) n += bits.OnesCount32(b[6]) n += bits.OnesCount32(b[7]) n += bits.OnesCount32(b[8]) n += bits.OnesCount32(b[9]) n += bits.OnesCount32(b[10]) n += bits.OnesCount32(b[11]) n += bits.OnesCount32(b[12]) n += bits.OnesCount32(b[13]) n += bits.OnesCount32(b[14]) n += bits.OnesCount32(b[15]) return n } func onescountAtomic(b *block) (n int) { n += bits.OnesCount32(atomic.LoadUint32(&b[0])) n += bits.OnesCount32(atomic.LoadUint32(&b[1])) n += bits.OnesCount32(atomic.LoadUint32(&b[2])) n += bits.OnesCount32(atomic.LoadUint32(&b[3])) n += bits.OnesCount32(atomic.LoadUint32(&b[4])) n += bits.OnesCount32(atomic.LoadUint32(&b[5])) n += bits.OnesCount32(atomic.LoadUint32(&b[6])) n += bits.OnesCount32(atomic.LoadUint32(&b[7])) n += bits.OnesCount32(atomic.LoadUint32(&b[8])) n += bits.OnesCount32(atomic.LoadUint32(&b[9])) n += bits.OnesCount32(atomic.LoadUint32(&b[10])) n += bits.OnesCount32(atomic.LoadUint32(&b[11])) n += bits.OnesCount32(atomic.LoadUint32(&b[12])) n += bits.OnesCount32(atomic.LoadUint32(&b[13])) n += bits.OnesCount32(atomic.LoadUint32(&b[14])) n += bits.OnesCount32(atomic.LoadUint32(&b[15])) return n } blobloom-0.7.1/sync.go000066400000000000000000000101211413725631400146300ustar00rootroot00000000000000// Copyright 2021 the Blobloom authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package blobloom import "sync/atomic" // A SyncFilter is a Bloom filter that can be accessed and updated // by multiple goroutines concurrently. // // A SyncFilter mostly behaves as a regular filter protected by a lock, // // type SyncFilter struct { // Filter // lock sync.Mutex // } // // with each method taking and releasing the lock, // but is implemented much more efficiently. // See the method descriptions for exceptions to the previous rule. type SyncFilter struct { b []block // Shards. k int // Number of hash functions required. } // NewSync constructs a Bloom filter with given numbers of bits and hash functions. // // The number of bits should be at least BlockBits; smaller values are silently // increased. // // The number of hashes reflects the number of hashes synthesized from the // single hash passed in by the client. It is silently increased to two if // a lower value is given. func NewSync(nbits uint64, nhashes int) *SyncFilter { return (*SyncFilter)(New(nbits, nhashes)) } // Add insert a key with hash value h into f. func (f *SyncFilter) Add(h uint64) { h1, h2 := uint32(h>>32), uint32(h) b := (*Filter)(f).getblock(h2) for i := 1; i < f.k; i++ { h1, h2 = doublehash(h1, h2, i) b.setbitAtomic(h1) } } // Cardinality estimates the number of distinct keys added to f. // // The estimate is most reliable when f is filled to roughly its capacity. // It gets worse as f gets more densely filled. When one of the blocks is // entirely filled, the estimate becomes +Inf. // // The return value is the maximum likelihood estimate of Papapetrou, Siberski // and Nejdl, summed over the blocks // (https://www.win.tue.nl/~opapapetrou/papers/Bloomfilters-DAPD.pdf). // // If other goroutines are concurrently adding keys, // the estimate may lie in between what would have been returned // before the concurrent updates started and what is returned // after the updates complete. func (f *SyncFilter) Cardinality() float64 { return (*Filter)(f).cardinality(onescountAtomic) } // Empty reports whether f contains no keys. // // If other goroutines are concurrently adding keys, // Empty may return a false positive. func (f *SyncFilter) Empty() bool { for i := 0; i < len(f.b); i++ { for j := 0; j < blockWords; j++ { if atomic.LoadUint32(&f.b[i][j]) != 0 { return false } } } return true } // Fill sets f to a completely full filter. // After Fill, Has returns true for any key. func (f *SyncFilter) Fill() { for i := 0; i < len(f.b); i++ { for j := 0; j < blockWords; j++ { atomic.StoreUint32(&f.b[i][j], ^uint32(0)) } } } // Has reports whether a key with hash value h has been added. // It may return a false positive. func (f *SyncFilter) Has(h uint64) bool { h1, h2 := uint32(h>>32), uint32(h) b := (*Filter)(f).getblock(h2) for i := 1; i < f.k; i++ { h1, h2 = doublehash(h1, h2, i) if !b.getbitAtomic(h1) { return false } } return true } // getbitAtomic reports whether bit (i modulo BlockBits) is set. func (b *block) getbitAtomic(i uint32) bool { bit := uint32(1) << (i % wordSize) x := atomic.LoadUint32(&(*b)[(i/wordSize)%blockWords]) return x&bit != 0 } // setbit sets bit (i modulo BlockBits) of b, atomically. func (b *block) setbitAtomic(i uint32) { bit := uint32(1) << (i % wordSize) p := &(*b)[(i/wordSize)%blockWords] for { old := atomic.LoadUint32(p) if old&bit != 0 { // Checking here instead of checking the return value from // the CAS is between 50% and 80% faster on the benchmark. return } atomic.CompareAndSwapUint32(p, old, old|bit) } } blobloom-0.7.1/sync_test.go000066400000000000000000000042441413725631400157000ustar00rootroot00000000000000// Copyright 2021 the Blobloom authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package blobloom import ( "math" "math/rand" "sync" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestSync(t *testing.T) { const ( nkeys = 1e4 nworkers = 4 ) var ( config = Config{Capacity: nkeys, FPRate: 1e-5} hashes = make([]uint64, nkeys) r = rand.New(rand.NewSource(0xaeb15)) ref = NewOptimized(config) ) for i := range hashes { h := r.Uint64() hashes[i] = h ref.Add(h) } card := ref.Cardinality() require.False(t, ref.Empty()) require.False(t, math.IsInf(card, 0)) check := func(f *SyncFilter) { t.Helper() assert.Equal(t, ref.b, f.b) assert.False(t, f.Empty()) for i := 0; i < 2e4; i++ { h := r.Uint64() assert.Equal(t, ref.Has(h), f.Has(h)) } assert.Equal(t, card, f.Cardinality()) } t.Run("all hashes", func(t *testing.T) { // Each worker adds all hashes to f. t.Parallel() f := NewSyncOptimized(config) assert.True(t, f.Empty()) var wg sync.WaitGroup wg.Add(nworkers) for i := 0; i < nworkers; i++ { go func() { for _, h := range hashes { f.Add(h) } wg.Done() }() } wg.Wait() check(f) }) t.Run("split hashes", func(t *testing.T) { // Hashes divided across workers. t.Parallel() var ( ch = make(chan uint64, nworkers) f = NewSyncOptimized(config) wg sync.WaitGroup ) wg.Add(nworkers) go func() { for _, h := range hashes { ch <- h } close(ch) }() for i := 0; i < nworkers; i++ { go func() { for h := range ch { f.Add(h) } wg.Done() }() } wg.Wait() check(f) }) } blobloom-0.7.1/test.sh000077500000000000000000000003361413725631400146520ustar00rootroot00000000000000#!/bin/sh set -e -x golangci-lint run . examples/* go test if [ "$(go env GOARCH)" = amd64 ]; then go test -tags nounsafe GOARCH=386 go test fi for e in examples/*; do (cd $e && go build && rm $(basename $e)) done