pax_global_header00006660000000000000000000000064136635300740014521gustar00rootroot0000000000000052 comment=9dc1b83dc0d8bf7d26ea69d6c6d7a11ebf6fbb89 fasthash-1.0.3/000077500000000000000000000000001366353007400133235ustar00rootroot00000000000000fasthash-1.0.3/.circleci/000077500000000000000000000000001366353007400151565ustar00rootroot00000000000000fasthash-1.0.3/.circleci/config.yml000066400000000000000000000005101366353007400171420ustar00rootroot00000000000000version: 2 jobs: build: working_directory: /go/src/github.com/segmentio/fasthash docker: - image: circleci/golang steps: - checkout - setup_remote_docker: { reusable: true, docker_layer_caching: true } - run: go get -v -t ./... - run: go vet ./... - run: go test -v -race ./... fasthash-1.0.3/.gitignore000066400000000000000000000004371366353007400153170ustar00rootroot00000000000000# Binaries for programs and plugins *.exe *.dll *.so *.dylib # Test binary, build with `go test -c` *.test # Output of the go coverage tool, specifically when used with LiteIDE *.out # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 .glide/ # Emacs *~ fasthash-1.0.3/LICENSE000066400000000000000000000020501366353007400143250ustar00rootroot00000000000000MIT License Copyright (c) 2017 Segment Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. fasthash-1.0.3/README.md000066400000000000000000000040761366353007400146110ustar00rootroot00000000000000# fasthash [![CircleCI](https://circleci.com/gh/segmentio/fasthash.svg?style=shield)](https://circleci.com/gh/segmentio/fasthash) [![Go Report Card](https://goreportcard.com/badge/github.com/segmentio/fasthash)](https://goreportcard.com/report/github.com/segmentio/fasthash) [![GoDoc](https://godoc.org/github.com/segmentio/fasthash?status.svg)](https://godoc.org/github.com/segmentio/fasthash) Go package porting the standard hashing algorithms to a more efficient implementation. ## Motivations Go has great support for hashing algorithms in the standard library, but the APIs are all exposed as interfaces, which means passing strings or byte slices to those require dynamic memory allocations. Hashing a string typically requires 2 allocations, one for the Hash value, and one to covert the string to a byte slice. This package attempts to solve this issue by exposing functions that implement string hashing algorithms and don't require dynamic memory alloations. ## Testing To ensure consistency between the `fasthash` package and the standard library, all tests must be implemented to run against the standard hash functions and validate that both packages produced the same results. ## Benchmarks The implementations also have to prove that they are more efficient in terms of CPU and memory usage than the functions found in the standard library. Here's an example with fnv-1a: ``` BenchmarkHash64/standard_hash_function 20000000 105.0 ns/op 342.31 MB/s 56 B/op 2 allocs/op BenchmarkHash64/hash_function 50000000 38.6 ns/op 932.35 MB/s 0 B/op 0 allocs/op ``` # Usage Example: FNV-1a ```go package main import ( "fmt" "github.com/segmentio/fasthash/fnv1a" ) func main() { // Hash a single string. h1 := fnv1a.HashString64("Hello World!") fmt.Println("FNV-1a hash of 'Hello World!':", h1) // Incrementally compute a hash value from a sequence of strings. h2 := fnv1a.Init64 h2 = fnv1a.AddString64(h2, "A") h2 = fnv1a.AddString64(h2, "B") h2 = fnv1a.AddString64(h2, "C") fmt.Println("FNV-1a hash of 'ABC':", h2) } ``` fasthash-1.0.3/fasthash.go000066400000000000000000000014301366353007400154510ustar00rootroot00000000000000package fasthash import ( "encoding/binary" "hash" ) // HashString64 makes a hashing function from the hashing algorithm returned by f. func HashString64(f func() hash.Hash64) func(string) uint64 { return func(s string) uint64 { h := f() h.Write([]byte(s)) return h.Sum64() } } // HashBytes64 makes a hashing function from the hashing algorithm returned by f. func HashBytes64(f func() hash.Hash64) func([]byte) uint64 { return func(b []byte) uint64 { h := f() h.Write(b) return h.Sum64() } } // HashUint64 makes a hashing function from the hashing algorithm return by f. func HashUint64(f func() hash.Hash64) func(uint64) uint64 { return func(u uint64) uint64 { b := [8]byte{} binary.BigEndian.PutUint64(b[:], u) h := f() h.Write(b[:]) return h.Sum64() } } fasthash-1.0.3/fasthash32.go000066400000000000000000000014301366353007400156160ustar00rootroot00000000000000package fasthash import ( "encoding/binary" "hash" ) // HashString32 makes a hashing function from the hashing algorithm returned by f. func HashString32(f func() hash.Hash32) func(string) uint32 { return func(s string) uint32 { h := f() h.Write([]byte(s)) return h.Sum32() } } // HashBytes32 makes a hashing function from the hashing algorithm returned by f. func HashBytes32(f func() hash.Hash32) func([]byte) uint32 { return func(s []byte) uint32 { h := f() h.Write(s) return h.Sum32() } } // HashUint32 makes a hashing function from the hashing algorithm return by f. func HashUint32(f func() hash.Hash32) func(uint32) uint32 { return func(u uint32) uint32 { b := [4]byte{} binary.BigEndian.PutUint32(b[:], u) h := f() h.Write(b[:]) return h.Sum32() } } fasthash-1.0.3/fasthashtest/000077500000000000000000000000001366353007400160245ustar00rootroot00000000000000fasthash-1.0.3/fasthashtest/fasthashtest.go000066400000000000000000000051551366353007400210620ustar00rootroot00000000000000package fasthashtest import ( "fmt" "strings" "testing" ) // TestHashString64 is the implementation of a test suite to verify the // behavior of a hashing algorithm. func TestHashString64(t *testing.T, name string, reference func(string) uint64, algorithm func(string) uint64) { t.Run(name, func(t *testing.T) { for _, s := range [...]string{"", "A", "Hello World!", "DAB45194-42CC-4106-AB9F-2447FA4D35C2", "你好吗"} { t.Run(s, func(t *testing.T) { if reference == nil { algorithm(s) } else { sum1 := reference(s) sum2 := algorithm(s) if sum1 != sum2 { t.Errorf("invalid hash, expected %x but got %x", sum1, sum2) } } }) } }) } // TestHashBytes64 is the implementation of a test suite to verify the // behavior of a hashing algorithm. func TestHashBytes64(t *testing.T, name string, reference func([]byte) uint64, algorithm func([]byte) uint64) { t.Run(name, func(t *testing.T) { for _, s := range [...]string{"", "A", "Hello World!", "DAB45194-42CC-4106-AB9F-2447FA4D35C2"} { t.Run(s, func(t *testing.T) { b := []byte(s) if reference == nil { algorithm(b) } else { sum1 := reference(b) sum2 := algorithm(b) if sum1 != sum2 { t.Errorf("invalid hash, expected %x but got %x", sum1, sum2) } } }) } }) } // TestHashUint64 is the implementation of a test suite to verify the // behavior of a hashing algorithm. func TestHashUint64(t *testing.T, name string, reference func(uint64) uint64, algorithm func(uint64) uint64) { t.Run(name, func(t *testing.T) { if reference == nil { algorithm(42) } else { sum1 := reference(42) sum2 := algorithm(42) if sum1 != sum2 { t.Errorf("invalid hash, expected %x but got %x", sum1, sum2) } } }) } // BenchmarkHashString64 is the implementation of a benchmark suite to compare // the CPU and memory efficiency of a hashing algorithm against a reference // implementation. func BenchmarkHashString64(b *testing.B, name string, reference func(string) uint64, algorithm func(string) uint64) { b.Run(name, func(b *testing.B) { if reference != nil { b.Run("reference", func(b *testing.B) { benchmark(b, reference) }) } b.Run("optimized", func(b *testing.B) { benchmark(b, algorithm) }) }) } var benchmarkStrings = [...]string{ "asdf", "hello world", "DAB45194-42CC-4106-AB9F-2447FA4D35C2", strings.Repeat("1234567890", 100), } func benchmark(b *testing.B, hash func(string) uint64) { for _, s := range benchmarkStrings { b.Run(fmt.Sprintf("strlen=%d", len(s)), func(b *testing.B) { for i := 0; i != b.N; i++ { hash(s) } b.SetBytes(int64(len(s))) }) } } fasthash-1.0.3/fasthashtest/fasthashtest32.go000066400000000000000000000047551366353007400212340ustar00rootroot00000000000000package fasthashtest import ( "fmt" "testing" ) // TestHashString32 is the implementation of a test suite to verify the // behavior of a hashing algorithm. func TestHashString32(t *testing.T, name string, reference func(string) uint32, algorithm func(string) uint32) { t.Run(name, func(t *testing.T) { for _, s := range [...]string{"", "A", "Hello World!", "DAB45194-42CC-4106-AB9F-2447FA4D35C2", "你好吗"} { t.Run(s, func(t *testing.T) { if reference == nil { algorithm(s) } else { sum1 := reference(s) sum2 := algorithm(s) if sum1 != sum2 { t.Errorf("invalid hash, expected %x but got %x", sum1, sum2) } } }) } }) } // TestHashBytes32 is the implementation of a test suite to verify the // behavior of a hashing algorithm. func TestHashBytes32(t *testing.T, name string, reference func([]byte) uint32, algorithm func([]byte) uint32) { t.Run(name, func(t *testing.T) { for _, s := range [...]string{"", "A", "Hello World!", "DAB45194-42CC-4106-AB9F-2447FA4D35C2"} { t.Run(s, func(t *testing.T) { b := []byte(s) if reference == nil { algorithm(b) } else { sum1 := reference(b) sum2 := algorithm(b) if sum1 != sum2 { t.Errorf("invalid hash, expected %x but got %x", sum1, sum2) } } }) } }) } // TestHashUint32 is the implementation of a test suite to verify the // behavior of a hashing algorithm. func TestHashUint32(t *testing.T, name string, reference func(uint32) uint32, algorithm func(uint32) uint32) { t.Run(name, func(t *testing.T) { if reference == nil { algorithm(42) } else { sum1 := reference(42) sum2 := algorithm(42) if sum1 != sum2 { t.Errorf("invalid hash, expected %x but got %x %v %v", sum1, sum2, sum1, sum2) } } }) } // BenchmarkHashString32 is the implementation of a benchmark suite to compare // the CPU and memory efficiency of a hashing algorithm against a reference // implementation. func BenchmarkHashString32(b *testing.B, name string, reference func(string) uint32, algorithm func(string) uint32) { b.Run(name, func(b *testing.B) { if reference != nil { b.Run("reference", func(b *testing.B) { benchmark32(b, reference) }) } b.Run("optimized", func(b *testing.B) { benchmark32(b, algorithm) }) }) } func benchmark32(b *testing.B, hash func(string) uint32) { for _, s := range benchmarkStrings { b.Run(fmt.Sprintf("strlen=%d", len(s)), func(b *testing.B) { for i := 0; i != b.N; i++ { hash(s) } b.SetBytes(int64(len(s))) }) } } fasthash-1.0.3/fnv1/000077500000000000000000000000001366353007400141755ustar00rootroot00000000000000fasthash-1.0.3/fnv1/hash.go000066400000000000000000000047031366353007400154530ustar00rootroot00000000000000package fnv1 const ( // FNV-1 offset64 = uint64(14695981039346656037) prime64 = uint64(1099511628211) // Init64 is what 64 bits hash values should be initialized with. Init64 = offset64 ) // HashString64 returns the hash of s. func HashString64(s string) uint64 { return AddString64(Init64, s) } // HashBytes64 returns the hash of u. func HashBytes64(b []byte) uint64 { return AddBytes64(Init64, b) } // HashUint64 returns the hash of u. func HashUint64(u uint64) uint64 { return AddUint64(Init64, u) } // AddString64 adds the hash of s to the precomputed hash value h. func AddString64(h uint64, s string) uint64 { for len(s) >= 8 { h = (h * prime64) ^ uint64(s[0]) h = (h * prime64) ^ uint64(s[1]) h = (h * prime64) ^ uint64(s[2]) h = (h * prime64) ^ uint64(s[3]) h = (h * prime64) ^ uint64(s[4]) h = (h * prime64) ^ uint64(s[5]) h = (h * prime64) ^ uint64(s[6]) h = (h * prime64) ^ uint64(s[7]) s = s[8:] } if len(s) >= 4 { h = (h * prime64) ^ uint64(s[0]) h = (h * prime64) ^ uint64(s[1]) h = (h * prime64) ^ uint64(s[2]) h = (h * prime64) ^ uint64(s[3]) s = s[4:] } if len(s) >= 2 { h = (h * prime64) ^ uint64(s[0]) h = (h * prime64) ^ uint64(s[1]) s = s[2:] } if len(s) > 0 { h = (h * prime64) ^ uint64(s[0]) } return h } // AddBytes64 adds the hash of b to the precomputed hash value h. func AddBytes64(h uint64, b []byte) uint64 { for len(b) >= 8 { h = (h * prime64) ^ uint64(b[0]) h = (h * prime64) ^ uint64(b[1]) h = (h * prime64) ^ uint64(b[2]) h = (h * prime64) ^ uint64(b[3]) h = (h * prime64) ^ uint64(b[4]) h = (h * prime64) ^ uint64(b[5]) h = (h * prime64) ^ uint64(b[6]) h = (h * prime64) ^ uint64(b[7]) b = b[8:] } if len(b) >= 4 { h = (h * prime64) ^ uint64(b[0]) h = (h * prime64) ^ uint64(b[1]) h = (h * prime64) ^ uint64(b[2]) h = (h * prime64) ^ uint64(b[3]) b = b[4:] } if len(b) >= 2 { h = (h * prime64) ^ uint64(b[0]) h = (h * prime64) ^ uint64(b[1]) b = b[2:] } if len(b) > 0 { h = (h * prime64) ^ uint64(b[0]) } return h } // AddUint64 adds the hash value of the 8 bytes of u to h. func AddUint64(h uint64, u uint64) uint64 { h = (h * prime64) ^ ((u >> 56) & 0xFF) h = (h * prime64) ^ ((u >> 48) & 0xFF) h = (h * prime64) ^ ((u >> 40) & 0xFF) h = (h * prime64) ^ ((u >> 32) & 0xFF) h = (h * prime64) ^ ((u >> 24) & 0xFF) h = (h * prime64) ^ ((u >> 16) & 0xFF) h = (h * prime64) ^ ((u >> 8) & 0xFF) h = (h * prime64) ^ ((u >> 0) & 0xFF) return h } fasthash-1.0.3/fnv1/hash32.go000066400000000000000000000044151366353007400156200ustar00rootroot00000000000000package fnv1 const ( // FNV-1 offset32 = uint32(2166136261) prime32 = uint32(16777619) // Init32 is what 32 bits hash values should be initialized with. Init32 = offset32 ) // HashString32 returns the hash of s. func HashString32(s string) uint32 { return AddString32(Init32, s) } // HashBytes32 returns the hash of u. func HashBytes32(b []byte) uint32 { return AddBytes32(Init32, b) } // HashUint32 returns the hash of u. func HashUint32(u uint32) uint32 { return AddUint32(Init32, u) } // AddString32 adds the hash of s to the precomputed hash value h. func AddString32(h uint32, s string) uint32 { for len(s) >= 8 { h = (h * prime32) ^ uint32(s[0]) h = (h * prime32) ^ uint32(s[1]) h = (h * prime32) ^ uint32(s[2]) h = (h * prime32) ^ uint32(s[3]) h = (h * prime32) ^ uint32(s[4]) h = (h * prime32) ^ uint32(s[5]) h = (h * prime32) ^ uint32(s[6]) h = (h * prime32) ^ uint32(s[7]) s = s[8:] } if len(s) >= 4 { h = (h * prime32) ^ uint32(s[0]) h = (h * prime32) ^ uint32(s[1]) h = (h * prime32) ^ uint32(s[2]) h = (h * prime32) ^ uint32(s[3]) s = s[4:] } if len(s) >= 2 { h = (h * prime32) ^ uint32(s[0]) h = (h * prime32) ^ uint32(s[1]) s = s[2:] } if len(s) > 0 { h = (h * prime32) ^ uint32(s[0]) } return h } // AddBytes32 adds the hash of b to the precomputed hash value h. func AddBytes32(h uint32, b []byte) uint32 { for len(b) >= 8 { h = (h * prime32) ^ uint32(b[0]) h = (h * prime32) ^ uint32(b[1]) h = (h * prime32) ^ uint32(b[2]) h = (h * prime32) ^ uint32(b[3]) h = (h * prime32) ^ uint32(b[4]) h = (h * prime32) ^ uint32(b[5]) h = (h * prime32) ^ uint32(b[6]) h = (h * prime32) ^ uint32(b[7]) b = b[8:] } if len(b) >= 4 { h = (h * prime32) ^ uint32(b[0]) h = (h * prime32) ^ uint32(b[1]) h = (h * prime32) ^ uint32(b[2]) h = (h * prime32) ^ uint32(b[3]) b = b[4:] } if len(b) >= 2 { h = (h * prime32) ^ uint32(b[0]) h = (h * prime32) ^ uint32(b[1]) b = b[2:] } if len(b) > 0 { h = (h * prime32) ^ uint32(b[0]) } return h } // AddUint32 adds the hash value of the 8 bytes of u to h. func AddUint32(h, u uint32) uint32 { h = (h * prime32) ^ ((u >> 24) & 0xFF) h = (h * prime32) ^ ((u >> 16) & 0xFF) h = (h * prime32) ^ ((u >> 8) & 0xFF) h = (h * prime32) ^ ((u >> 0) & 0xFF) return h } fasthash-1.0.3/fnv1/hash32_test.go000066400000000000000000000010571366353007400166560ustar00rootroot00000000000000package fnv1 import ( "hash/fnv" "testing" "github.com/segmentio/fasthash" "github.com/segmentio/fasthash/fasthashtest" ) func TestHash32(t *testing.T) { fasthashtest.TestHashString32(t, "fnv1", fasthash.HashString32(fnv.New32), HashString32) fasthashtest.TestHashBytes32(t, "fnv1", fasthash.HashBytes32(fnv.New32), HashBytes32) fasthashtest.TestHashUint32(t, "fnv1", fasthash.HashUint32(fnv.New32), HashUint32) } func BenchmarkHash32(b *testing.B) { fasthashtest.BenchmarkHashString32(b, "fnv1", fasthash.HashString32(fnv.New32), HashString32) } fasthash-1.0.3/fnv1/hash_test.go000066400000000000000000000010571366353007400165110ustar00rootroot00000000000000package fnv1 import ( "hash/fnv" "testing" "github.com/segmentio/fasthash" "github.com/segmentio/fasthash/fasthashtest" ) func TestHash64(t *testing.T) { fasthashtest.TestHashString64(t, "fnv1", fasthash.HashString64(fnv.New64), HashString64) fasthashtest.TestHashBytes64(t, "fnv1", fasthash.HashBytes64(fnv.New64), HashBytes64) fasthashtest.TestHashUint64(t, "fnv1", fasthash.HashUint64(fnv.New64), HashUint64) } func BenchmarkHash64(b *testing.B) { fasthashtest.BenchmarkHashString64(b, "fnv1", fasthash.HashString64(fnv.New64), HashString64) } fasthash-1.0.3/fnv1a/000077500000000000000000000000001366353007400143365ustar00rootroot00000000000000fasthash-1.0.3/fnv1a/hash.go000066400000000000000000000055101366353007400156110ustar00rootroot00000000000000package fnv1a const ( // FNV-1a offset64 = uint64(14695981039346656037) prime64 = uint64(1099511628211) // Init64 is what 64 bits hash values should be initialized with. Init64 = offset64 ) // HashString64 returns the hash of s. func HashString64(s string) uint64 { return AddString64(Init64, s) } // HashBytes64 returns the hash of u. func HashBytes64(b []byte) uint64 { return AddBytes64(Init64, b) } // HashUint64 returns the hash of u. func HashUint64(u uint64) uint64 { return AddUint64(Init64, u) } // AddString64 adds the hash of s to the precomputed hash value h. func AddString64(h uint64, s string) uint64 { /* This is an unrolled version of this algorithm: for _, c := range s { h = (h ^ uint64(c)) * prime64 } It seems to be ~1.5x faster than the simple loop in BenchmarkHash64: - BenchmarkHash64/hash_function-4 30000000 56.1 ns/op 642.15 MB/s 0 B/op 0 allocs/op - BenchmarkHash64/hash_function-4 50000000 38.6 ns/op 932.35 MB/s 0 B/op 0 allocs/op */ for len(s) >= 8 { h = (h ^ uint64(s[0])) * prime64 h = (h ^ uint64(s[1])) * prime64 h = (h ^ uint64(s[2])) * prime64 h = (h ^ uint64(s[3])) * prime64 h = (h ^ uint64(s[4])) * prime64 h = (h ^ uint64(s[5])) * prime64 h = (h ^ uint64(s[6])) * prime64 h = (h ^ uint64(s[7])) * prime64 s = s[8:] } if len(s) >= 4 { h = (h ^ uint64(s[0])) * prime64 h = (h ^ uint64(s[1])) * prime64 h = (h ^ uint64(s[2])) * prime64 h = (h ^ uint64(s[3])) * prime64 s = s[4:] } if len(s) >= 2 { h = (h ^ uint64(s[0])) * prime64 h = (h ^ uint64(s[1])) * prime64 s = s[2:] } if len(s) > 0 { h = (h ^ uint64(s[0])) * prime64 } return h } // AddBytes64 adds the hash of b to the precomputed hash value h. func AddBytes64(h uint64, b []byte) uint64 { for len(b) >= 8 { h = (h ^ uint64(b[0])) * prime64 h = (h ^ uint64(b[1])) * prime64 h = (h ^ uint64(b[2])) * prime64 h = (h ^ uint64(b[3])) * prime64 h = (h ^ uint64(b[4])) * prime64 h = (h ^ uint64(b[5])) * prime64 h = (h ^ uint64(b[6])) * prime64 h = (h ^ uint64(b[7])) * prime64 b = b[8:] } if len(b) >= 4 { h = (h ^ uint64(b[0])) * prime64 h = (h ^ uint64(b[1])) * prime64 h = (h ^ uint64(b[2])) * prime64 h = (h ^ uint64(b[3])) * prime64 b = b[4:] } if len(b) >= 2 { h = (h ^ uint64(b[0])) * prime64 h = (h ^ uint64(b[1])) * prime64 b = b[2:] } if len(b) > 0 { h = (h ^ uint64(b[0])) * prime64 } return h } // AddUint64 adds the hash value of the 8 bytes of u to h. func AddUint64(h uint64, u uint64) uint64 { h = (h ^ ((u >> 56) & 0xFF)) * prime64 h = (h ^ ((u >> 48) & 0xFF)) * prime64 h = (h ^ ((u >> 40) & 0xFF)) * prime64 h = (h ^ ((u >> 32) & 0xFF)) * prime64 h = (h ^ ((u >> 24) & 0xFF)) * prime64 h = (h ^ ((u >> 16) & 0xFF)) * prime64 h = (h ^ ((u >> 8) & 0xFF)) * prime64 h = (h ^ ((u >> 0) & 0xFF)) * prime64 return h } fasthash-1.0.3/fnv1a/hash32.go000066400000000000000000000044171366353007400157630ustar00rootroot00000000000000package fnv1a const ( // FNV-1a offset32 = uint32(2166136261) prime32 = uint32(16777619) // Init32 is what 32 bits hash values should be initialized with. Init32 = offset32 ) // HashString32 returns the hash of s. func HashString32(s string) uint32 { return AddString32(Init32, s) } // HashBytes32 returns the hash of u. func HashBytes32(b []byte) uint32 { return AddBytes32(Init32, b) } // HashUint32 returns the hash of u. func HashUint32(u uint32) uint32 { return AddUint32(Init32, u) } // AddString32 adds the hash of s to the precomputed hash value h. func AddString32(h uint32, s string) uint32 { for len(s) >= 8 { h = (h ^ uint32(s[0])) * prime32 h = (h ^ uint32(s[1])) * prime32 h = (h ^ uint32(s[2])) * prime32 h = (h ^ uint32(s[3])) * prime32 h = (h ^ uint32(s[4])) * prime32 h = (h ^ uint32(s[5])) * prime32 h = (h ^ uint32(s[6])) * prime32 h = (h ^ uint32(s[7])) * prime32 s = s[8:] } if len(s) >= 4 { h = (h ^ uint32(s[0])) * prime32 h = (h ^ uint32(s[1])) * prime32 h = (h ^ uint32(s[2])) * prime32 h = (h ^ uint32(s[3])) * prime32 s = s[4:] } if len(s) >= 2 { h = (h ^ uint32(s[0])) * prime32 h = (h ^ uint32(s[1])) * prime32 s = s[2:] } if len(s) > 0 { h = (h ^ uint32(s[0])) * prime32 } return h } // AddBytes32 adds the hash of b to the precomputed hash value h. func AddBytes32(h uint32, b []byte) uint32 { for len(b) >= 8 { h = (h ^ uint32(b[0])) * prime32 h = (h ^ uint32(b[1])) * prime32 h = (h ^ uint32(b[2])) * prime32 h = (h ^ uint32(b[3])) * prime32 h = (h ^ uint32(b[4])) * prime32 h = (h ^ uint32(b[5])) * prime32 h = (h ^ uint32(b[6])) * prime32 h = (h ^ uint32(b[7])) * prime32 b = b[8:] } if len(b) >= 4 { h = (h ^ uint32(b[0])) * prime32 h = (h ^ uint32(b[1])) * prime32 h = (h ^ uint32(b[2])) * prime32 h = (h ^ uint32(b[3])) * prime32 b = b[4:] } if len(b) >= 2 { h = (h ^ uint32(b[0])) * prime32 h = (h ^ uint32(b[1])) * prime32 b = b[2:] } if len(b) > 0 { h = (h ^ uint32(b[0])) * prime32 } return h } // AddUint32 adds the hash value of the 8 bytes of u to h. func AddUint32(h, u uint32) uint32 { h = (h ^ ((u >> 24) & 0xFF)) * prime32 h = (h ^ ((u >> 16) & 0xFF)) * prime32 h = (h ^ ((u >> 8) & 0xFF)) * prime32 h = (h ^ ((u >> 0) & 0xFF)) * prime32 return h } fasthash-1.0.3/fnv1a/hash32_test.go000066400000000000000000000010701366353007400170120ustar00rootroot00000000000000package fnv1a import ( "hash/fnv" "testing" "github.com/segmentio/fasthash" "github.com/segmentio/fasthash/fasthashtest" ) func TestHash32(t *testing.T) { fasthashtest.TestHashString32(t, "fnv1a", fasthash.HashString32(fnv.New32a), HashString32) fasthashtest.TestHashBytes32(t, "fnv1a", fasthash.HashBytes32(fnv.New32a), HashBytes32) fasthashtest.TestHashUint32(t, "fnv1a", fasthash.HashUint32(fnv.New32a), HashUint32) } func BenchmarkHash32(b *testing.B) { fasthashtest.BenchmarkHashString32(b, "fnv1a", fasthash.HashString32(fnv.New32a), HashString32) } fasthash-1.0.3/fnv1a/hash_test.go000066400000000000000000000010701366353007400166450ustar00rootroot00000000000000package fnv1a import ( "hash/fnv" "testing" "github.com/segmentio/fasthash" "github.com/segmentio/fasthash/fasthashtest" ) func TestHash64(t *testing.T) { fasthashtest.TestHashString64(t, "fnv1a", fasthash.HashString64(fnv.New64a), HashString64) fasthashtest.TestHashBytes64(t, "fnv1a", fasthash.HashBytes64(fnv.New64a), HashBytes64) fasthashtest.TestHashUint64(t, "fnv1a", fasthash.HashUint64(fnv.New64a), HashUint64) } func BenchmarkHash64(b *testing.B) { fasthashtest.BenchmarkHashString64(b, "fnv1a", fasthash.HashString64(fnv.New64a), HashString64) } fasthash-1.0.3/go.mod000066400000000000000000000000561366353007400144320ustar00rootroot00000000000000module github.com/segmentio/fasthash go 1.11 fasthash-1.0.3/jody/000077500000000000000000000000001366353007400142705ustar00rootroot00000000000000fasthash-1.0.3/jody/hash.go000066400000000000000000000062501366353007400155450ustar00rootroot00000000000000package jody import ( "reflect" "unsafe" ) const ( shift = 11 constant = 0x1f3d5b79 // Init64 is what 64 bits hash values should be initialized with. Init64 = uint64(0) ) var mask64 = [...]uint64{ 0x0000000000000000, 0x00000000000000ff, 0x000000000000ffff, 0x0000000000ffffff, 0x00000000ffffffff, 0x000000ffffffffff, 0x0000ffffffffffff, 0x00ffffffffffffff, 0xffffffffffffffff, } // HashString64 returns the hash of s. func HashString64(s string) uint64 { return AddString64(Init64, s) } // HashBytes64 returns the hash of u. func HashBytes64(b []byte) uint64 { return AddBytes64(Init64, b) } // HashUint64 returns the hash of u. func HashUint64(u uint64) uint64 { return AddUint64(Init64, u) } // AddString64 adds the hash of s to the precomputed hash value h. func AddString64(h uint64, s string) uint64 { /* This is an implementation of the jody hashing algorithm as found here: - https://github.com/jbruchon/jodyhash It was revisited a bit and only the 64 bit version is available for now, but it's indeed really fast: - BenchmarkHash64/jody/algorithm-4 100000000 11.8 ns/op 3050.83 MB/s 0 B/op 0 allocs/op Here's what the reference implementation looks like: hash_t hash = start_hash; hash_t element; hash_t partial_salt; size_t len; len = count / sizeof(hash_t); for (; len > 0; len--) { element = *data; hash += element; hash += constant; hash = (hash << shift) | hash >> (sizeof(hash_t) * 8 - shift); hash ^= element; hash = (hash << shift) | hash >> (sizeof(hash_t) * 8 - shift); hash ^= constant; hash += element; data++; } len = count & (sizeof(hash_t) - 1); if (len) { partial_salt = constant & tail_mask[len]; element = *data & tail_mask[len]; hash += element; hash += partial_salt; hash = (hash << shift) | hash >> (sizeof(hash_t) * 8 - shift); hash ^= element; hash = (hash << shift) | hash >> (sizeof(hash_t) * 8 - shift); hash ^= partial_salt; hash += element; } return hash; */ r := *(*reflect.StringHeader)(unsafe.Pointer(&s)) p := unsafe.Pointer((*reflect.StringHeader)(unsafe.Pointer(&s)).Data) for n := r.Len / 8; n != 0; n-- { v := *(*uint64)(p) h = h + v + constant h = (h<>(64-shift)) ^ v h = ((h<>(64-shift)) ^ constant) + v p = unsafe.Pointer(uintptr(p) + 8) } if n := (r.Len & 7); n != 0 { c := constant & mask64[n] v := uint64(0) off := uint(0) if 0 != (n & 4) { v += uint64(*(*uint32)(p)) off += 32 p = unsafe.Pointer(uintptr(p) + 4) } if 0 != (n & 2) { v += uint64(*(*uint16)(p)) << off off += 16 p = unsafe.Pointer(uintptr(p) + 2) } if 0 != (n & 1) { v += uint64(*(*uint8)(p)) << off } h = h + v + c h = (h<>(64-shift)) ^ v h = ((h<>(64-shift)) ^ c) + v } return h } // AddBytes64 adds the hash of b to the precomputed hash value h. func AddBytes64(h uint64, b []byte) uint64 { return AddString64(h, *(*string)(unsafe.Pointer(&b))) } // AddUint64 adds the hash value of the 8 bytes of u to h. func AddUint64(h uint64, u uint64) uint64 { h = h + u + constant h = (h<>(64-shift)) ^ u h = ((h<>(64-shift)) ^ constant) + u return h } fasthash-1.0.3/jody/hash_test.go000066400000000000000000000022731366353007400166050ustar00rootroot00000000000000package jody import ( "fmt" "testing" "github.com/segmentio/fasthash/fasthashtest" ) func TestHash64(t *testing.T) { // I couldn't find a reference implementation in Go, so this is a hacky // test that checks for values generated from the jodyhash command line // utility. referenceString64 := func(s string) uint64 { switch s { case "": return 0x0000000000000000 case "A": return 0x000000002e8208ba case "Hello World!": return 0x60be57b5a53eb1c7 case "DAB45194-42CC-4106-AB9F-2447FA4D35C2": return 0x587861d2b41e1997 case "你好吗": return 0x944cd5c16171eca3 default: panic("test not implemented: " + s) } } referenceByte64 := func(b []byte) uint64 { return referenceString64(string(b)) } referenceUint64 := func(u uint64) uint64 { if u != 42 { panic(fmt.Sprint("test not implemented:", u)) } return 0x0007cf56f7fc0ba3 } fasthashtest.TestHashString64(t, "jody", referenceString64, HashString64) fasthashtest.TestHashBytes64(t, "jody", referenceByte64, HashBytes64) fasthashtest.TestHashUint64(t, "jody", referenceUint64, HashUint64) } func BenchmarkHash64(b *testing.B) { fasthashtest.BenchmarkHashString64(b, "jody", nil, HashString64) }