Skip to content

Commit

Permalink
fixing golangci-lint issues
Browse files Browse the repository at this point in the history
  • Loading branch information
matthiasthomas committed Jan 30, 2025
1 parent 429f33c commit 1772bb6
Show file tree
Hide file tree
Showing 2 changed files with 33 additions and 2 deletions.
13 changes: 12 additions & 1 deletion claude.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
_ "embed"
"encoding/base64"
"encoding/json"
"fmt"
"strconv"
"strings"
)
Expand Down Expand Up @@ -44,7 +45,17 @@ func NewClaude() (*Codec, error) {
return nil, bErr
}

mergeableRanks[string(t)] = uint(i * offset)
if offset < 0 {
return nil, fmt.Errorf("negative value not allowed: %d", offset)
}

product := i * offset

if i < 0 || product < 0 || product > (1<<32-1) {
return nil, fmt.Errorf("integer overflow in calculation: %d * %d", i, offset)
}

mergeableRanks[string(t)] = uint(product)
}

return &Codec{
Expand Down
22 changes: 21 additions & 1 deletion codec.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"strconv"
"strings"
Expand Down Expand Up @@ -76,6 +77,10 @@ func CovertVocabBPEAndEncoderJSONToMergeableBPERanks(vocabBPE io.Reader, encoder
bpeRanks := make(map[string]uint)

for i, b := range rankToIntByte {
if i < 0 {
return nil, fmt.Errorf("negative value not allowed: %d", i)
}

key := string(b)
bpeRanks[key] = uint(i)
}
Expand All @@ -96,6 +101,11 @@ func CovertVocabBPEAndEncoderJSONToMergeableBPERanks(vocabBPE io.Reader, encoder
first := decodeDataGym(merge[0])
second := decodeDataGym(merge[1])
key := string(append(first, second...))

if n < 0 {
return nil, fmt.Errorf("negative value not allowed: %d", n)
}

bpeRanks[key] = uint(n)
n++
}
Expand All @@ -120,7 +130,13 @@ func CovertVocabBPEAndEncoderJSONToMergeableBPERanks(vocabBPE io.Reader, encoder
result = append(result, dataGymByteToByte[string(r)])
}

encoderLoaded[string(result)] = uint(v.(float64))
if val, ok := v.(float64); ok {
if val < 0 || val > float64(1<<32-1) {
return nil, fmt.Errorf("value out of uint range: %f", val)
}

encoderLoaded[string(result)] = uint(val)
}
}

// delete these two special tokens if present, since they're not
Expand Down Expand Up @@ -171,6 +187,10 @@ func ConvertToMergeableBPERanks(bpe io.Reader) (map[string]uint, error) {
return nil, err
}

if rank < 0 {
return nil, fmt.Errorf("negative value not allowed: %d", rank)
}

bpeRanks[string(token)] = uint(rank)
}

Expand Down

0 comments on commit 1772bb6

Please sign in to comment.