pax_global_header00006660000000000000000000000064126736003770014525gustar00rootroot0000000000000052 comment=f051bb7f1d1aaf1b5a665d74fb6b0217712c69f7 xxHash-0.1.1/000077500000000000000000000000001267360037700127675ustar00rootroot00000000000000xxHash-0.1.1/.travis.yml000066400000000000000000000004141267360037700150770ustar00rootroot00000000000000language: go go: - 1.4 - 1.5 script: - go test -cpu=2 github.com/pierrec/xxHash/xxHash32 - go test -cpu=2 github.com/pierrec/xxHash/xxHash64 - go test -cpu=2 -race github.com/pierrec/xxHash/xxHash32 - go test -cpu=2 -race github.com/pierrec/xxHash/xxHash64 xxHash-0.1.1/LICENSE000066400000000000000000000027051267360037700140000ustar00rootroot00000000000000Copyright (c) 2014, Pierre Curto All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of xxHash nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. xxHash-0.1.1/README.md000066400000000000000000000016501267360037700142500ustar00rootroot00000000000000[![godoc](https://godoc.org/github.com/pierrec/xxHash?status.png)](https://godoc.org/github.com/pierrec/xxHash) [![Build Status](https://travis-ci.org/pierrec/xxHash.svg?branch=master)](https://travis-ci.org/pierrec/xxHash) # Pure Go implementation of xxHash (32 and 64 bits versions) ## Synopsis xxHash is a very fast hashing algorithm (see the details [here](https://github.com/Cyan4973/xxHash/)). This package implements xxHash in pure [Go](http://www.golang.com). ## Usage This package follows the hash interfaces (hash.Hash32 and hash.Hash64). ```go import ( "fmt" "github.com/pierrec/xxHash/xxHash32" ) x := xxHash32.New(0xCAFE) // hash.Hash32 x.Write([]byte("abc")) x.Write([]byte("def")) fmt.Printf("%x\n", x.Sum32()) x.Reset() x.Write([]byte("abc")) fmt.Printf("%x\n", x.Sum32()) ``` ## Command line utility A simple command line utility is provided to hash files content under the xxhsum directory. xxHash-0.1.1/xxHash32/000077500000000000000000000000001267360037700143775ustar00rootroot00000000000000xxHash-0.1.1/xxHash32/example_test.go000066400000000000000000000006411267360037700174210ustar00rootroot00000000000000package xxHash32_test import ( "bytes" "fmt" "github.com/pierrec/xxHash/xxHash32" ) func ExampleNew() { buf := bytes.NewBufferString("this is a test") x := xxHash32.New(0xCAFE) x.Write(buf.Bytes()) fmt.Printf("%x\n", x.Sum32()) // Output: bb4f02bc } func ExampleChecksum() { buf := bytes.NewBufferString("this is a test") fmt.Printf("%x\n", xxHash32.Checksum(buf.Bytes(), 0xCAFE)) // Output: bb4f02bc } xxHash-0.1.1/xxHash32/xxHash32.go000066400000000000000000000126721267360037700163460ustar00rootroot00000000000000// Package xxHash32 implements the very fast xxHash hashing algorithm (32 bits version). // (https://github.com/Cyan4973/xxHash/) package xxHash32 import "hash" const ( prime32_1 = 2654435761 prime32_2 = 2246822519 prime32_3 = 3266489917 prime32_4 = 668265263 prime32_5 = 374761393 ) type xxHash struct { seed uint32 v1 uint32 v2 uint32 v3 uint32 v4 uint32 totalLen uint64 buf [16]byte bufused int } // New returns a new Hash32 instance. func New(seed uint32) hash.Hash32 { xxh := &xxHash{seed: seed} xxh.Reset() return xxh } // Sum appends the current hash to b and returns the resulting slice. // It does not change the underlying hash state. func (xxh xxHash) Sum(b []byte) []byte { h32 := xxh.Sum32() return append(b, byte(h32), byte(h32>>8), byte(h32>>16), byte(h32>>24)) } // Reset resets the Hash to its initial state. func (xxh *xxHash) Reset() { xxh.v1 = xxh.seed + prime32_1 + prime32_2 xxh.v2 = xxh.seed + prime32_2 xxh.v3 = xxh.seed xxh.v4 = xxh.seed - prime32_1 xxh.totalLen = 0 xxh.bufused = 0 } // Size returns the number of bytes returned by Sum(). func (xxh *xxHash) Size() int { return 4 } // BlockSize gives the minimum number of bytes accepted by Write(). func (xxh *xxHash) BlockSize() int { return 1 } // Write adds input bytes to the Hash. // It never returns an error. func (xxh *xxHash) Write(input []byte) (int, error) { n := len(input) m := xxh.bufused xxh.totalLen += uint64(n) r := len(xxh.buf) - m if n < r { copy(xxh.buf[m:], input) xxh.bufused += len(input) return n, nil } p := 0 if m > 0 { // some data left from previous update copy(xxh.buf[xxh.bufused:], input[:r]) xxh.bufused += len(input) - r // fast rotl(13) p32 := xxh.v1 + (uint32(xxh.buf[p+3])<<24|uint32(xxh.buf[p+2])<<16|uint32(xxh.buf[p+1])<<8|uint32(xxh.buf[p]))*prime32_2 xxh.v1 = (p32<<13 | p32>>19) * prime32_1 p += 4 p32 = xxh.v2 + (uint32(xxh.buf[p+3])<<24|uint32(xxh.buf[p+2])<<16|uint32(xxh.buf[p+1])<<8|uint32(xxh.buf[p]))*prime32_2 xxh.v2 = (p32<<13 | p32>>19) * prime32_1 p += 4 p32 = xxh.v3 + (uint32(xxh.buf[p+3])<<24|uint32(xxh.buf[p+2])<<16|uint32(xxh.buf[p+1])<<8|uint32(xxh.buf[p]))*prime32_2 xxh.v3 = (p32<<13 | p32>>19) * prime32_1 p += 4 p32 = xxh.v4 + (uint32(xxh.buf[p+3])<<24|uint32(xxh.buf[p+2])<<16|uint32(xxh.buf[p+1])<<8|uint32(xxh.buf[p]))*prime32_2 xxh.v4 = (p32<<13 | p32>>19) * prime32_1 p = r xxh.bufused = 0 } for n := n - 16; p <= n; { p32 := xxh.v1 + (uint32(input[p+3])<<24|uint32(input[p+2])<<16|uint32(input[p+1])<<8|uint32(input[p]))*prime32_2 xxh.v1 = (p32<<13 | p32>>19) * prime32_1 p += 4 p32 = xxh.v2 + (uint32(input[p+3])<<24|uint32(input[p+2])<<16|uint32(input[p+1])<<8|uint32(input[p]))*prime32_2 xxh.v2 = (p32<<13 | p32>>19) * prime32_1 p += 4 p32 = xxh.v3 + (uint32(input[p+3])<<24|uint32(input[p+2])<<16|uint32(input[p+1])<<8|uint32(input[p]))*prime32_2 xxh.v3 = (p32<<13 | p32>>19) * prime32_1 p += 4 p32 = xxh.v4 + (uint32(input[p+3])<<24|uint32(input[p+2])<<16|uint32(input[p+1])<<8|uint32(input[p]))*prime32_2 xxh.v4 = (p32<<13 | p32>>19) * prime32_1 p += 4 } copy(xxh.buf[xxh.bufused:], input[p:]) xxh.bufused += len(input) - p return n, nil } // Sum32 returns the 32 bits Hash value. func (xxh *xxHash) Sum32() uint32 { h32 := uint32(xxh.totalLen) if xxh.totalLen >= 16 { h32 += ((xxh.v1 << 1) | (xxh.v1 >> 31)) + ((xxh.v2 << 7) | (xxh.v2 >> 25)) + ((xxh.v3 << 12) | (xxh.v3 >> 20)) + ((xxh.v4 << 18) | (xxh.v4 >> 14)) } else { h32 += xxh.seed + prime32_5 } p := 0 n := xxh.bufused for n := n - 4; p <= n; p += 4 { h32 += (uint32(xxh.buf[p+3])<<24 | uint32(xxh.buf[p+2])<<16 | uint32(xxh.buf[p+1])<<8 | uint32(xxh.buf[p])) * prime32_3 h32 = ((h32 << 17) | (h32 >> 15)) * prime32_4 } for ; p < n; p++ { h32 += uint32(xxh.buf[p]) * prime32_5 h32 = ((h32 << 11) | (h32 >> 21)) * prime32_1 } h32 ^= h32 >> 15 h32 *= prime32_2 h32 ^= h32 >> 13 h32 *= prime32_3 h32 ^= h32 >> 16 return h32 } // Checksum returns the 32bits Hash value. func Checksum(input []byte, seed uint32) uint32 { n := len(input) h32 := uint32(n) if n < 16 { h32 += seed + prime32_5 } else { v1 := seed + prime32_1 + prime32_2 v2 := seed + prime32_2 v3 := seed v4 := seed - prime32_1 p := 0 for p <= n-16 { v1 += (uint32(input[p+3])<<24 | uint32(input[p+2])<<16 | uint32(input[p+1])<<8 | uint32(input[p])) * prime32_2 v1 = (v1<<13 | v1>>19) * prime32_1 p += 4 v2 += (uint32(input[p+3])<<24 | uint32(input[p+2])<<16 | uint32(input[p+1])<<8 | uint32(input[p])) * prime32_2 v2 = (v2<<13 | v2>>19) * prime32_1 p += 4 v3 += (uint32(input[p+3])<<24 | uint32(input[p+2])<<16 | uint32(input[p+1])<<8 | uint32(input[p])) * prime32_2 v3 = (v3<<13 | v3>>19) * prime32_1 p += 4 v4 += (uint32(input[p+3])<<24 | uint32(input[p+2])<<16 | uint32(input[p+1])<<8 | uint32(input[p])) * prime32_2 v4 = (v4<<13 | v4>>19) * prime32_1 p += 4 } input = input[p:] n -= p h32 += ((v1 << 1) | (v1 >> 31)) + ((v2 << 7) | (v2 >> 25)) + ((v3 << 12) | (v3 >> 20)) + ((v4 << 18) | (v4 >> 14)) } p := 0 for p <= n-4 { h32 += (uint32(input[p+3])<<24 | uint32(input[p+2])<<16 | uint32(input[p+1])<<8 | uint32(input[p])) * prime32_3 h32 = ((h32 << 17) | (h32 >> 15)) * prime32_4 p += 4 } for p < n { h32 += uint32(input[p]) * prime32_5 h32 = ((h32 << 11) | (h32 >> 21)) * prime32_1 p++ } h32 ^= h32 >> 15 h32 *= prime32_2 h32 ^= h32 >> 13 h32 *= prime32_3 h32 ^= h32 >> 16 return h32 } xxHash-0.1.1/xxHash32/xxHash32_test.go000066400000000000000000000067361267360037700174110ustar00rootroot00000000000000package xxHash32_test import ( "encoding/binary" "hash/crc32" "hash/fnv" "testing" "github.com/pierrec/xxHash/xxHash32" ) type test struct { sum uint32 data, printable string } var testdata = []test{ {0x02cc5d05, "", ""}, {0x550d7456, "a", ""}, {0x4999fc53, "ab", ""}, {0x32d153ff, "abc", ""}, {0xa3643705, "abcd", ""}, {0x9738f19b, "abcde", ""}, {0x8b7cd587, "abcdef", ""}, {0x9dd093b3, "abcdefg", ""}, {0x0bb3c6bb, "abcdefgh", ""}, {0xd03c13fd, "abcdefghi", ""}, {0x8b988cfe, "abcdefghij", ""}, {0x9d2d8b62, "abcdefghijklmnop", ""}, {0x42ae804d, "abcdefghijklmnopqrstuvwxyz0123456789", ""}, {0x62b4ed00, "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.", ""}, } func init() { for i := range testdata { d := &testdata[i] if len(d.data) > 20 { d.printable = d.data[:20] } else { d.printable = d.data } } } func TestBlockSize(t *testing.T) { xxh := xxHash32.New(0) if s := xxh.BlockSize(); s <= 0 { t.Errorf("invalid BlockSize: %d", s) } } func TestSize(t *testing.T) { xxh := xxHash32.New(0) if s := xxh.Size(); s != 4 { t.Errorf("invalid Size: got %d expected 4", s) } } func TestData(t *testing.T) { for i, td := range testdata { xxh := xxHash32.New(0) data := []byte(td.data) xxh.Write(data) if h := xxh.Sum32(); h != td.sum { t.Errorf("test %d: xxh32(%s)=0x%x expected 0x%x", i, td.printable, h, td.sum) t.FailNow() } if h := xxHash32.Checksum(data, 0); h != td.sum { t.Errorf("test %d: xxh32(%s)=0x%x expected 0x%x", i, td.printable, h, td.sum) t.FailNow() } } } func TestSplitData(t *testing.T) { for i, td := range testdata { xxh := xxHash32.New(0) data := []byte(td.data) l := len(data) / 2 xxh.Write(data[0:l]) xxh.Write(data[l:]) h := xxh.Sum32() if h != td.sum { t.Errorf("test %d: xxh32(%s)=0x%x expected 0x%x", i, td.printable, h, td.sum) t.FailNow() } } } func TestSum(t *testing.T) { for i, td := range testdata { xxh := xxHash32.New(0) data := []byte(td.data) xxh.Write(data) b := xxh.Sum(data) if h := binary.LittleEndian.Uint32(b[len(data):]); h != td.sum { t.Errorf("test %d: xxh32(%s)=0x%x expected 0x%x", i, td.printable, h, td.sum) t.FailNow() } } } func TestReset(t *testing.T) { xxh := xxHash32.New(0) for i, td := range testdata { xxh.Write([]byte(td.data)) h := xxh.Sum32() if h != td.sum { t.Errorf("test %d: xxh32(%s)=0x%x expected 0x%x", i, td.data[:40], h, td.sum) t.FailNow() } xxh.Reset() } } /////////////////////////////////////////////////////////////////////////////// // Benchmarks // var testdata1 = []byte(testdata[len(testdata)-1].data) func Benchmark_XXH32(b *testing.B) { h := xxHash32.New(0) for n := 0; n < b.N; n++ { h.Write(testdata1) h.Sum32() h.Reset() } } func Benchmark_XXH32_Checksum(b *testing.B) { for n := 0; n < b.N; n++ { xxHash32.Checksum(testdata1, 0) } } func Benchmark_CRC32(b *testing.B) { t := crc32.MakeTable(0) for i := 0; i < b.N; i++ { crc32.Checksum(testdata1, t) } } func Benchmark_Fnv32(b *testing.B) { h := fnv.New32() for i := 0; i < b.N; i++ { h.Write(testdata1) h.Sum32() h.Reset() } } xxHash-0.1.1/xxHash64/000077500000000000000000000000001267360037700144045ustar00rootroot00000000000000xxHash-0.1.1/xxHash64/example_test.go000066400000000000000000000006611267360037700174300ustar00rootroot00000000000000package xxHash64_test import ( "bytes" "fmt" "github.com/pierrec/xxHash/xxHash64" ) func ExampleNew() { buf := bytes.NewBufferString("this is a test") x := xxHash64.New(0xCAFE) x.Write(buf.Bytes()) fmt.Printf("%x\n", x.Sum64()) // Output: 4228c3215949e862 } func ExampleChecksum() { buf := bytes.NewBufferString("this is a test") fmt.Printf("%x\n", xxHash64.Checksum(buf.Bytes(), 0xCAFE)) // Output: 4228c3215949e862 } xxHash-0.1.1/xxHash64/xxHash64.go000066400000000000000000000203421267360037700163510ustar00rootroot00000000000000// Package xxHash64 implements the very fast xxHash hashing algorithm (64 bits version). // (https://github.com/Cyan4973/xxHash/) package xxHash64 import "hash" const ( prime64_1 = 11400714785074694791 prime64_2 = 14029467366897019727 prime64_3 = 1609587929392839161 prime64_4 = 9650029242287828579 prime64_5 = 2870177450012600261 ) type xxHash struct { seed uint64 v1 uint64 v2 uint64 v3 uint64 v4 uint64 totalLen uint64 buf [32]byte bufused int } // New returns a new Hash64 instance. func New(seed uint64) hash.Hash64 { xxh := &xxHash{seed: seed} xxh.Reset() return xxh } // Sum appends the current hash to b and returns the resulting slice. // It does not change the underlying hash state. func (xxh xxHash) Sum(b []byte) []byte { h64 := xxh.Sum64() return append(b, byte(h64), byte(h64>>8), byte(h64>>16), byte(h64>>24), byte(h64>>32), byte(h64>>40), byte(h64>>48), byte(h64>>56)) } // Reset resets the Hash to its initial state. func (xxh *xxHash) Reset() { xxh.v1 = xxh.seed + prime64_1 + prime64_2 xxh.v2 = xxh.seed + prime64_2 xxh.v3 = xxh.seed xxh.v4 = xxh.seed - prime64_1 xxh.totalLen = 0 xxh.bufused = 0 } // Size returns the number of bytes returned by Sum(). func (xxh *xxHash) Size() int { return 8 } // BlockSize gives the minimum number of bytes accepted by Write(). func (xxh *xxHash) BlockSize() int { return 1 } // Write adds input bytes to the Hash. // It never returns an error. func (xxh *xxHash) Write(input []byte) (int, error) { n := len(input) m := xxh.bufused xxh.totalLen += uint64(n) r := len(xxh.buf) - m if n < r { copy(xxh.buf[m:], input) xxh.bufused += len(input) return n, nil } p := 0 if m > 0 { // some data left from previous update copy(xxh.buf[xxh.bufused:], input[:r]) xxh.bufused += len(input) - r // fast rotl(31) p64 := xxh.v1 + (uint64(xxh.buf[p+7])<<56|uint64(xxh.buf[p+6])<<48|uint64(xxh.buf[p+5])<<40|uint64(xxh.buf[p+4])<<32|uint64(xxh.buf[p+3])<<24|uint64(xxh.buf[p+2])<<16|uint64(xxh.buf[p+1])<<8|uint64(xxh.buf[p]))*prime64_2 xxh.v1 = (p64<<31 | p64>>33) * prime64_1 p += 8 p64 = xxh.v2 + (uint64(xxh.buf[p+7])<<56|uint64(xxh.buf[p+6])<<48|uint64(xxh.buf[p+5])<<40|uint64(xxh.buf[p+4])<<32|uint64(xxh.buf[p+3])<<24|uint64(xxh.buf[p+2])<<16|uint64(xxh.buf[p+1])<<8|uint64(xxh.buf[p]))*prime64_2 xxh.v2 = (p64<<31 | p64>>33) * prime64_1 p += 8 p64 = xxh.v3 + (uint64(xxh.buf[p+7])<<56|uint64(xxh.buf[p+6])<<48|uint64(xxh.buf[p+5])<<40|uint64(xxh.buf[p+4])<<32|uint64(xxh.buf[p+3])<<24|uint64(xxh.buf[p+2])<<16|uint64(xxh.buf[p+1])<<8|uint64(xxh.buf[p]))*prime64_2 xxh.v3 = (p64<<31 | p64>>33) * prime64_1 p += 8 p64 = xxh.v4 + (uint64(xxh.buf[p+7])<<56|uint64(xxh.buf[p+6])<<48|uint64(xxh.buf[p+5])<<40|uint64(xxh.buf[p+4])<<32|uint64(xxh.buf[p+3])<<24|uint64(xxh.buf[p+2])<<16|uint64(xxh.buf[p+1])<<8|uint64(xxh.buf[p]))*prime64_2 xxh.v4 = (p64<<31 | p64>>33) * prime64_1 p = r xxh.bufused = 0 } for n := n - 32; p <= n; { p64 := xxh.v1 + (uint64(input[p+7])<<56|uint64(input[p+6])<<48|uint64(input[p+5])<<40|uint64(input[p+4])<<32|uint64(input[p+3])<<24|uint64(input[p+2])<<16|uint64(input[p+1])<<8|uint64(input[p]))*prime64_2 xxh.v1 = (p64<<31 | p64>>33) * prime64_1 p += 8 p64 = xxh.v2 + (uint64(input[p+7])<<56|uint64(input[p+6])<<48|uint64(input[p+5])<<40|uint64(input[p+4])<<32|uint64(input[p+3])<<24|uint64(input[p+2])<<16|uint64(input[p+1])<<8|uint64(input[p]))*prime64_2 xxh.v2 = (p64<<31 | p64>>33) * prime64_1 p += 8 p64 = xxh.v3 + (uint64(input[p+7])<<56|uint64(input[p+6])<<48|uint64(input[p+5])<<40|uint64(input[p+4])<<32|uint64(input[p+3])<<24|uint64(input[p+2])<<16|uint64(input[p+1])<<8|uint64(input[p]))*prime64_2 xxh.v3 = (p64<<31 | p64>>33) * prime64_1 p += 8 p64 = xxh.v4 + (uint64(input[p+7])<<56|uint64(input[p+6])<<48|uint64(input[p+5])<<40|uint64(input[p+4])<<32|uint64(input[p+3])<<24|uint64(input[p+2])<<16|uint64(input[p+1])<<8|uint64(input[p]))*prime64_2 xxh.v4 = (p64<<31 | p64>>33) * prime64_1 p += 8 } copy(xxh.buf[xxh.bufused:], input[p:]) xxh.bufused += len(input) - p return n, nil } // Sum64 returns the 64bits Hash value. func (xxh *xxHash) Sum64() uint64 { var h64 uint64 if xxh.totalLen >= 32 { h64 = ((xxh.v1 << 1) | (xxh.v1 >> 63)) + ((xxh.v2 << 7) | (xxh.v2 >> 57)) + ((xxh.v3 << 12) | (xxh.v3 >> 52)) + ((xxh.v4 << 18) | (xxh.v4 >> 46)) xxh.v1 *= prime64_2 h64 ^= ((xxh.v1 << 31) | (xxh.v1 >> 33)) * prime64_1 h64 = h64*prime64_1 + prime64_4 xxh.v2 *= prime64_2 h64 ^= ((xxh.v2 << 31) | (xxh.v2 >> 33)) * prime64_1 h64 = h64*prime64_1 + prime64_4 xxh.v3 *= prime64_2 h64 ^= ((xxh.v3 << 31) | (xxh.v3 >> 33)) * prime64_1 h64 = h64*prime64_1 + prime64_4 xxh.v4 *= prime64_2 h64 ^= ((xxh.v4 << 31) | (xxh.v4 >> 33)) * prime64_1 h64 = h64*prime64_1 + prime64_4 + xxh.totalLen } else { h64 = xxh.seed + prime64_5 + xxh.totalLen } p := 0 n := xxh.bufused for n := n - 8; p <= n; p += 8 { p64 := (uint64(xxh.buf[p+7])<<56 | uint64(xxh.buf[p+6])<<48 | uint64(xxh.buf[p+5])<<40 | uint64(xxh.buf[p+4])<<32 | uint64(xxh.buf[p+3])<<24 | uint64(xxh.buf[p+2])<<16 | uint64(xxh.buf[p+1])<<8 | uint64(xxh.buf[p])) * prime64_2 h64 ^= ((p64 << 31) | (p64 >> 33)) * prime64_1 h64 = ((h64<<27)|(h64>>37))*prime64_1 + prime64_4 } if p+4 <= n { h64 ^= (uint64(xxh.buf[p+3])<<24 | uint64(xxh.buf[p+2])<<16 | uint64(xxh.buf[p+1])<<8 | uint64(xxh.buf[p])) * prime64_1 h64 = ((h64<<23)|(h64>>41))*prime64_2 + prime64_3 p += 4 } for ; p < n; p++ { h64 ^= uint64(xxh.buf[p]) * prime64_5 h64 = ((h64 << 11) | (h64 >> 53)) * prime64_1 } h64 ^= h64 >> 33 h64 *= prime64_2 h64 ^= h64 >> 29 h64 *= prime64_3 h64 ^= h64 >> 32 return h64 } // Checksum returns the 64bits Hash value. func Checksum(input []byte, seed uint64) uint64 { n := len(input) var h64 uint64 if n >= 32 { v1 := seed + prime64_1 + prime64_2 v2 := seed + prime64_2 v3 := seed v4 := seed - prime64_1 p := 0 for n := n - 32; p <= n; { p64 := v1 + (uint64(input[p+7])<<56|uint64(input[p+6])<<48|uint64(input[p+5])<<40|uint64(input[p+4])<<32|uint64(input[p+3])<<24|uint64(input[p+2])<<16|uint64(input[p+1])<<8|uint64(input[p]))*prime64_2 v1 = (p64<<31 | p64>>33) * prime64_1 p += 8 p64 = v2 + (uint64(input[p+7])<<56|uint64(input[p+6])<<48|uint64(input[p+5])<<40|uint64(input[p+4])<<32|uint64(input[p+3])<<24|uint64(input[p+2])<<16|uint64(input[p+1])<<8|uint64(input[p]))*prime64_2 v2 = (p64<<31 | p64>>33) * prime64_1 p += 8 p64 = v3 + (uint64(input[p+7])<<56|uint64(input[p+6])<<48|uint64(input[p+5])<<40|uint64(input[p+4])<<32|uint64(input[p+3])<<24|uint64(input[p+2])<<16|uint64(input[p+1])<<8|uint64(input[p]))*prime64_2 v3 = (p64<<31 | p64>>33) * prime64_1 p += 8 p64 = v4 + (uint64(input[p+7])<<56|uint64(input[p+6])<<48|uint64(input[p+5])<<40|uint64(input[p+4])<<32|uint64(input[p+3])<<24|uint64(input[p+2])<<16|uint64(input[p+1])<<8|uint64(input[p]))*prime64_2 v4 = (p64<<31 | p64>>33) * prime64_1 p += 8 } h64 = ((v1 << 1) | (v1 >> 63)) + ((v2 << 7) | (v2 >> 57)) + ((v3 << 12) | (v3 >> 52)) + ((v4 << 18) | (v4 >> 46)) v1 *= prime64_2 h64 ^= ((v1 << 31) | (v1 >> 33)) * prime64_1 h64 = h64*prime64_1 + prime64_4 v2 *= prime64_2 h64 ^= ((v2 << 31) | (v2 >> 33)) * prime64_1 h64 = h64*prime64_1 + prime64_4 v3 *= prime64_2 h64 ^= ((v3 << 31) | (v3 >> 33)) * prime64_1 h64 = h64*prime64_1 + prime64_4 v4 *= prime64_2 h64 ^= ((v4 << 31) | (v4 >> 33)) * prime64_1 h64 = h64*prime64_1 + prime64_4 + uint64(n) input = input[p:] n -= p } else { h64 = seed + prime64_5 + uint64(n) } p := 0 for n := n - 8; p <= n; p += 8 { p64 := (uint64(input[p+7])<<56 | uint64(input[p+6])<<48 | uint64(input[p+5])<<40 | uint64(input[p+4])<<32 | uint64(input[p+3])<<24 | uint64(input[p+2])<<16 | uint64(input[p+1])<<8 | uint64(input[p])) * prime64_2 h64 ^= ((p64 << 31) | (p64 >> 33)) * prime64_1 h64 = ((h64<<27)|(h64>>37))*prime64_1 + prime64_4 } if p+4 <= n { h64 ^= (uint64(input[p+3])<<24 | uint64(input[p+2])<<16 | uint64(input[p+1])<<8 | uint64(input[p])) * prime64_1 h64 = ((h64<<23)|(h64>>41))*prime64_2 + prime64_3 p += 4 } for ; p < n; p++ { h64 ^= uint64(input[p]) * prime64_5 h64 = ((h64 << 11) | (h64 >> 53)) * prime64_1 } h64 ^= h64 >> 33 h64 *= prime64_2 h64 ^= h64 >> 29 h64 *= prime64_3 h64 ^= h64 >> 32 return h64 } xxHash-0.1.1/xxHash64/xxHash64_test.go000066400000000000000000000071361267360037700174160ustar00rootroot00000000000000package xxHash64_test import ( "encoding/binary" "hash/crc64" "hash/fnv" "testing" "github.com/pierrec/xxHash/xxHash64" ) type test struct { sum uint64 data, printable string } var testdata = []test{ {0xef46db3751d8e999, "", ""}, {0xd24ec4f1a98c6e5b, "a", ""}, {0x65f708ca92d04a61, "ab", ""}, {0x44bc2cf5ad770999, "abc", ""}, {0xde0327b0d25d92cc, "abcd", ""}, {0x07e3670c0c8dc7eb, "abcde", ""}, {0xfa8afd82c423144d, "abcdef", ""}, {0x1860940e2902822d, "abcdefg", ""}, {0x3ad351775b4634b7, "abcdefgh", ""}, {0x27f1a34fdbb95e13, "abcdefghi", ""}, {0xd6287a1de5498bb2, "abcdefghij", ""}, {0xbf2cd639b4143b80, "abcdefghijklmnopqrstuvwxyz012345", ""}, {0x64f23ecf1609b766, "abcdefghijklmnopqrstuvwxyz0123456789", ""}, {0xc5a8b11443765630, "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.", ""}, } func init() { for i := range testdata { d := &testdata[i] if len(d.data) > 20 { d.printable = d.data[:20] } else { d.printable = d.data } } } func TestBlockSize(t *testing.T) { xxh := xxHash64.New(0) if s := xxh.BlockSize(); s <= 0 { t.Errorf("invalid BlockSize: %d", s) } } func TestSize(t *testing.T) { xxh := xxHash64.New(0) if s := xxh.Size(); s != 8 { t.Errorf("invalid Size: got %d expected 8", s) } } func TestData(t *testing.T) { for i, td := range testdata { xxh := xxHash64.New(0) data := []byte(td.data) xxh.Write(data) if h := xxh.Sum64(); h != td.sum { t.Errorf("test %d: xxh64(%s)=0x%x expected 0x%x", i, td.printable, h, td.sum) t.FailNow() } if h := xxHash64.Checksum(data, 0); h != td.sum { t.Errorf("test %d: xxh64(%s)=0x%x expected 0x%x", i, td.printable, h, td.sum) t.FailNow() } } } func TestSplitData(t *testing.T) { for i, td := range testdata { xxh := xxHash64.New(0) data := []byte(td.data) l := len(data) / 2 xxh.Write(data[0:l]) xxh.Write(data[l:]) h := xxh.Sum64() if h != td.sum { t.Errorf("test %d: xxh64(%s)=0x%x expected 0x%x", i, td.printable, h, td.sum) t.FailNow() } } } func TestSum(t *testing.T) { for i, td := range testdata { xxh := xxHash64.New(0) data := []byte(td.data) xxh.Write(data) b := xxh.Sum(data) if h := binary.LittleEndian.Uint64(b[len(data):]); h != td.sum { t.Errorf("test %d: xxh64(%s)=0x%x expected 0x%x", i, td.printable, h, td.sum) t.FailNow() } } } func TestReset(t *testing.T) { xxh := xxHash64.New(0) for i, td := range testdata { xxh.Write([]byte(td.data)) h := xxh.Sum64() if h != td.sum { t.Errorf("test %d: xxh64(%s)=0x%x expected 0x%x", i, td.printable, h, td.sum) t.FailNow() } xxh.Reset() } } /////////////////////////////////////////////////////////////////////////////// // Benchmarks // var testdata1 = []byte(testdata[len(testdata)-1].data) func Benchmark_XXH64(b *testing.B) { h := xxHash64.New(0) for n := 0; n < b.N; n++ { h.Write(testdata1) h.Sum64() h.Reset() } } func Benchmark_XXH64_Checksum(b *testing.B) { for n := 0; n < b.N; n++ { xxHash64.Checksum(testdata1, 0) } } func Benchmark_CRC64(b *testing.B) { t := crc64.MakeTable(0) for i := 0; i < b.N; i++ { crc64.Checksum(testdata1, t) } } func Benchmark_Fnv64(b *testing.B) { h := fnv.New64() for i := 0; i < b.N; i++ { h.Write(testdata1) h.Sum64() h.Reset() } } xxHash-0.1.1/xxhsum/000077500000000000000000000000001267360037700143235ustar00rootroot00000000000000xxHash-0.1.1/xxhsum/main.go000066400000000000000000000016211267360037700155760ustar00rootroot00000000000000// Command line interface to the xxHash32 and xxHash64 packages. // Usage: // xxHash [-mode 0] [-seed 123] filename1 [filename2...] // where // mode: hash mode (0=32bits, 1=64bits) (default=1) // seed: seed to be used (default=0) package main import ( "flag" "fmt" "hash" "io" "os" "github.com/pierrec/xxHash/xxHash32" "github.com/pierrec/xxHash/xxHash64" ) func main() { seed := flag.Uint64("seed", 0, "seed value") mode := flag.Int("mode", 1, "hash mode: 0=32bits, 1=64bits") flag.Parse() var xxh hash.Hash if *mode == 0 { xxh = xxHash32.New(uint32(*seed)) } else { xxh = xxHash64.New(*seed) } // Process each file in sequence for _, filename := range flag.Args() { inputFile, err := os.Open(filename) if err != nil { continue } if _, err := io.Copy(xxh, inputFile); err == nil { fmt.Printf("%x %s\n", xxh.Sum(nil), filename) } inputFile.Close() xxh.Reset() } }