From 9fbdf4100785a96f09cb8050984202b188ac050c Mon Sep 17 00:00:00 2001 From: askuy Date: Sat, 16 Nov 2024 21:25:31 +0800 Subject: [PATCH] snowflake --- .github/workflows/go.yml | 29 +++++++++ esnowflake.go | 63 +++++++++++++------ esnowflake_test.go | 127 +++++++++++++++++++++++++++++++++++++++ go.mod | 11 ++++ 4 files changed, 212 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/go.yml create mode 100644 esnowflake_test.go create mode 100644 go.mod diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..196a4f7 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,29 @@ +name: Go + +on: + push: + pull_request: + +jobs: + + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.21 + - name: Build + run: go build -v ./... + - name: Test + run: go test -v -race $(go list ./... | grep -v /examples/) -coverprofile=coverage.txt -covermode=atomic + - name: CodeCov + uses: codecov/codecov-action@v1 + with: + # token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos + files: ./coverage.txt + flags: unittests # optional + name: codecov-umbrella # optional + fail_ci_if_error: true # optional (default = false) + verbose: true # optional (default = false) \ No newline at end of file diff --git a/esnowflake.go b/esnowflake.go index 7f0829d..1bbe1f3 100644 --- a/esnowflake.go +++ b/esnowflake.go @@ -1,15 +1,31 @@ /* +Random +* 1 41 65 128 +* +---------------------------------------------+----------------------------+--------------------------------------------------------------------------+ +* | timestamp(ms) | worker info | random number | +* +---------------------------------------------+----------------------------+--------------------------------------------------------------------------+ +* | 00000000 00000000 00000000 00000000 00000000 | 00000000 00000000 00000000 | 00000000 00000000 00000000 00000000 00000000 00000000 | 00000000 00000000 | +* +---------------------------------------------+----------------------------+--------------------------------------------------------------------------+ +* +* 1. 40 位时间截(毫秒级)，注意这是时间截的差值（当前时间截 - 开始时间截)。可以使用约 34 年: (1L << 40) / (1000L * 60 * 60 * 24 * 365) = 34。（2020-2054） +* 2. 24 位 worker info 数据，适应 k8s 环境。 +* 3. 64 随机数 +*/ + +/* +Sequence * 1 41 65 113 128 * +---------------------------------------------+----------------------------+------------------------------------------------------+-------------------+ -* | timestamp(ms) | worker info | random number | sequence | +* | timestamp(ms) | worker info | random number | sequence | * +---------------------------------------------+----------------------------+------------------------------------------------------+-------------------+ -* | 0000000000 0000000000 0000000000 0000000000 | 0000000000 0000000000 0000 | 0000000000 0000000000 0000000000 0000000000 00000000 | 00000000 00000000 | +* | 00000000 00000000 00000000 00000000 00000000 | 00000000 00000000 00000000 | 00000000 00000000 00000000 00000000 00000000 00000000 | 00000000 00000000 | * +---------------------------------------------+----------------------------+------------------------------------------------------+-------------------+ * * 1. 40 位时间截(毫秒级)，注意这是时间截的差值（当前时间截 - 开始时间截)。可以使用约 34 年: (1L << 40) / (1000L * 60 * 60 * 24 * 365) = 34。（2020-2054） * 2. 24 位 worker info 数据，适应 k8s 环境。 * 3. 64 随机数 - */ +* 4. 16 位 sequence +*/ package esnowflake @@ -29,18 +45,17 @@ const ( workerInfoBits = uint(24) // 机器 ip 所占的位数 ) -const randPoolSequenceSize = 8 * 6 -const randPoolRandomSize = 8 * 6 +const randPoolSequenceRandomSize = 6 * 256 +const randPoolRandomSize = 8 * 64 * 3 var ( - rander = rand.Reader // random function - poolSequencePos = randPoolSequenceSize // protected with poolMu - poolSequence [randPoolSequenceSize]byte // protected with poolMu - poolRandomPos = randPoolRandomSize // protected with poolMu - poolRandom [randPoolRandomSize]byte // protected with poolMu - //mask = [3]byte{123, 45, 67} - sequenceBits = uint(16) // 序列所占的位数 - sequenceMask = int64(-1 ^ (-1 << sequenceBits)) // + rander = rand.Reader // random function + poolSequenceRandomPos = randPoolSequenceRandomSize // protected with poolMu + poolSequenceRandom [randPoolSequenceRandomSize]byte // protected with poolMu + poolRandomPos = randPoolRandomSize // protected with poolMu + poolRandom [randPoolRandomSize]byte // protected with poolMu + sequenceBits = uint(16) // 序列所占的位数 + sequenceMask = int64(-1 ^ (-1 << sequenceBits)) // ) @@ -54,6 +69,9 @@ type Config struct { sequence int64 } +// New create a new snowflake node with a unique worker id. +// ip 你的机器 ip +// mask1, mask2, mask3 你自己填的随机数，用于混淆 ip func New(ip string, mask1, mask2, mask3 uint8) *Config { b := net.ParseIP(ip).To4() if b == nil { @@ -71,11 +89,15 @@ func New(ip string, mask1, mask2, mask3 uint8) *Config { return &obj } +// GenerateByRandom 生成一个唯一的 id， 这个性能比较好，只有非常低的被碰撞概率，我们认为可以忽略不计 func (s *Config) GenerateByRandom() string { buf := make([]byte, 16) s.Lock() now := time.Now().UnixNano() / 1000000 if poolRandomPos == randPoolRandomSize { + // 生成48个字节的随机数 + // 下面在buf中，每次取8个字节 + // 如果 poolRandomPos 到达最大值，重新生成随机数，并将 poolRandomPos 置为 0 _, err := io.ReadFull(rander, poolRandom[:]) if err != nil { s.Unlock() @@ -107,18 +129,23 @@ func (s *Config) GenerateBySequence() string { } s.timestamp = now - if poolSequencePos == randPoolSequenceSize { - _, err := io.ReadFull(rander, poolSequence[:]) + // 生成48个字节的随机数 + // 下面在buf中，每次取6个字节 + // 如果 poolSequenceRandomPos 到达最大值，重新生成随机数，并将 poolSequenceRandomPos 置为 0 + if poolSequenceRandomPos == randPoolSequenceRandomSize { + _, err := io.ReadFull(rander, poolSequenceRandom[:]) if err != nil { s.Unlock() panic(err) } - poolSequencePos = 0 + poolSequenceRandomPos = 0 } copy(buf[:5], Uint64ToBytes(uint64(now-twepoch) << workerInfoBits)[:5]) copy(buf[5:8], s.ip) - copy(buf[8:14], poolSequence[poolSequencePos:(poolSequencePos+7)]) - poolSequencePos += 7 + // 随机数 + copy(buf[8:14], poolSequenceRandom[poolSequenceRandomPos:(poolSequenceRandomPos+6)]) + poolSequenceRandomPos += 6 + // sequence copy(buf[14:], Uint64ToBytes(uint64(s.sequence))) s.Unlock() return base64.RawURLEncoding.EncodeToString(buf) diff --git a/esnowflake_test.go b/esnowflake_test.go new file mode 100644 index 0000000..d0040d5 --- /dev/null +++ b/esnowflake_test.go @@ -0,0 +1,127 @@ +package esnowflake + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestConfigWithValidIP_ReturnsConfig(t *testing.T) { + config := New("192.168.1.1", 1, 2, 3) + if config == nil { + t.Error("Expected non-nil config") + } +} + +func TestConfigWithInvalidIP_Panics(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Error("Expected panic for invalid IP") + } + }() + New("invalid_ip", 1, 2, 3) +} + +func TestConfigWithValidIP_SetsMaskedIP(t *testing.T) { + config := New("192.168.1.2", 1, 2, 3) + expectedIP := []byte{168 ^ 1, 1 ^ 2, 2 ^ 3} + for i, b := range expectedIP { + if config.ip[i] != b { + t.Errorf("Expected masked IP byte %d to be %d, got %d", i, b, config.ip[i]) + } + } +} + +func TestConfigWithValidIP_GetIp(t *testing.T) { + config := New("192.168.1.2", 1, 2, 3) + encode := config.GenerateByRandom() + ip := config.GetIP(encode) + assert.Equal(t, "xxx.168.1.2", ip) + +} + +func TestGenerateByRandom_ReturnsUniqueIDs(t *testing.T) { + config := New("192.168.1.1", 1, 2, 3) + id1 := config.GenerateByRandom() + id2 := config.GenerateByRandom() + if id1 == id2 { + t.Errorf("Expected unique IDs, but got %s and %s", id1, id2) + } +} + +func TestGenerateByRandom_HandlesPoolRefill(t *testing.T) { + config := New("192.168.1.1", 1, 2, 3) + for i := 0; i < 100; i++ { + config.GenerateByRandom() + } +} + +func TestGenerateByRandom_GetTime(t *testing.T) { + config := New("192.168.1.1", 1, 2, 3) + encode := config.GenerateByRandom() + encodeTime := config.GetTime(encode) + fmt.Printf("time--------------->"+"%+v\n", encodeTime) +} + +func TestGenerateByRandom_HandlesLocking(t *testing.T) { + config := New("192.168.1.1", 1, 2, 3) + done := make(chan bool) + go func() { + config.GenerateByRandom() + done <- true + }() + select { + case <-done: + case <-time.After(1 * time.Second): + t.Error("GenerateByRandom did not return within 1 second") + } +} + +func TestGenerateBySequence_ReturnsUniqueIDs(t *testing.T) { + config := New("192.168.1.1", 1, 2, 3) + id1 := config.GenerateBySequence() + id2 := config.GenerateBySequence() + if id1 == id2 { + t.Errorf("Expected unique IDs, but got %s and %s", id1, id2) + } +} + +func TestGenerateBySequence_HandlesSequenceOverflow(t *testing.T) { + config := New("192.168.1.1", 1, 2, 3) + config.timestamp = time.Now().UnixNano() / 1000000 + config.sequence = sequenceMask + id := config.GenerateBySequence() + if id == "" { + t.Error("Expected non-empty ID on sequence overflow") + } +} + +func TestGenerateBySequence_HandlesLocking(t *testing.T) { + config := New("192.168.1.1", 1, 2, 3) + done := make(chan bool) + go func() { + config.GenerateBySequence() + done <- true + }() + select { + case <-done: + case <-time.After(1 * time.Second): + t.Error("GenerateBySequence did not return within 1 second") + } +} + +func BenchmarkRandomAndSequence(b *testing.B) { + obj := New("192.168.1.1", 1, 2, 3) + b.Run("Random", func(b *testing.B) { + for i := 0; i < b.N; i++ { + obj.GenerateByRandom() + } + }) + b.Run("Sequence", func(b *testing.B) { + for i := 0; i < b.N; i++ { + obj.GenerateBySequence() + } + }) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..51268d7 --- /dev/null +++ b/go.mod @@ -0,0 +1,11 @@ +module github.com/ego-component/esnowflake + +go 1.21.1 + +require github.com/stretchr/testify v1.9.0 + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +)