From c66ffdabe2b1f304f376a8f1bf06a5fd32e77263 Mon Sep 17 00:00:00 2001 From: WSRer <1749094641@qq.com> Date: Tue, 15 Oct 2024 17:19:04 +0800 Subject: [PATCH 1/2] feat: distract index --- go.mod | 2 +- index.go | 51 ++++++++++++++++++++++ main.go | 114 ++++++++++++++++++------------------------------ storage_test.go | 61 ++++++++++++++++++++++++++ 4 files changed, 155 insertions(+), 73 deletions(-) create mode 100644 index.go create mode 100644 storage_test.go diff --git a/go.mod b/go.mod index 916c96b..ef68d40 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module simple-hash-kv-storage +module gitee.com/asphodelus_dev/simple-hashtable-storage go 1.23.0 diff --git a/index.go b/index.go new file mode 100644 index 0000000..6370d5f --- /dev/null +++ b/index.go @@ -0,0 +1,51 @@ +package main + +import ( + "fmt" + "iter" +) + +type Index interface { + Get(key []byte) (uint64, error) + Set(key []byte, value uint64) error + Delete(key []byte) error + Iterator() iter.Seq2[[]byte, uint64] +} + +var _ Index = (*DefaultIndex)(nil) + +type DefaultIndex struct { + inner map[string]uint64 +} + +func (d *DefaultIndex) Iterator() iter.Seq2[[]byte, uint64] { + return func(yield func([]byte, uint64) bool) { + for k, v := range d.inner { + // 将字符串键转换为[]byte + keyBytes := []byte(k) + if !yield(keyBytes, v) { + return + } + } + } +} + +func (d *DefaultIndex) Get(key []byte) (uint64, error) { + u, exists := d.inner[string(key)] + if !exists { + return 0, fmt.Errorf("key not found") + } + return u, nil +} + +func (d *DefaultIndex) Set(key []byte, value uint64) error { + d.inner[string(key)] = value + return nil +} + +func (d *DefaultIndex) Delete(key []byte) error { + if _, exists := d.inner[string(key)]; exists { + delete(d.inner, string(key)) + } + return nil +} diff --git a/main.go b/main.go index 4793549..95f2665 100644 --- a/main.go +++ b/main.go @@ -50,12 +50,18 @@ type DataPage struct { type StorageEngine struct { sync.RWMutex // 读写锁用于并发控制 wal *wal.WAL - indexes map[string]uint64 // 索引：键 -> 页号 + indexes Index // 索引：键 -> 页号 file *os.File } +type Options struct { + DirPath string + DataIndex Index +} + // NewStorageEngine 创建一个新的存储引擎实例，并从磁盘加载数据 -func NewStorageEngine(dir string) (*StorageEngine, error) { +func NewStorageEngine(opt Options) (*StorageEngine, error) { + dir := opt.DirPath // wal用于记录对indexes的操作，重启后可以从wal恢复indexes walOptions := wal.DefaultOptions walOptions.DirPath = dir @@ -69,8 +75,11 @@ func NewStorageEngine(dir string) (*StorageEngine, error) { if err != nil { return nil, err } + if opt.DataIndex == nil { + opt.DataIndex = &DefaultIndex{inner: make(map[string]uint64)} + } engine := &StorageEngine{ - indexes: make(map[string]uint64), + indexes: opt.DataIndex, wal: w, file: file, } @@ -110,9 +119,16 @@ func (se *StorageEngine) loadFromDisk() error { switch rec.Type { case WalOpInsert: - se.indexes[string(rec.Key)] = rec.PageID + err = se.indexes.Set(rec.Key, rec.PageID) + if err != nil { + return err + } case WalOpDelete: - delete(se.indexes, string(rec.Key)) + + err = se.indexes.Delete(rec.Key) + if err != nil { + return err + } } } return nil @@ -210,7 +226,7 @@ func (se *StorageEngine) findOrAllocatePage(record Record) (uint64, error) { recordSize := uint64(len(record.Key) + len(record.Value) + 16) // 16 bytes for key and value sizes // 查找可用的页 - for _, pageID := range se.indexes { + for _, pageID := range se.indexes.Iterator() { page, exists := se.getPage(pageID) if !exists { continue @@ -287,7 +303,10 @@ func (se *StorageEngine) Insert(key, value []byte) error { } page.UserRecords = append(page.UserRecords, Record{Key: key, Value: value}) - se.indexes[string(key)] = pageID + err = se.indexes.Set(key, pageID) + if err != nil { + return err + } err = se.savePage(pageID, *page) if err != nil { @@ -318,7 +337,10 @@ func (se *StorageEngine) Modify(key, value []byte) error { } } - se.indexes[string(key)] = pageID + err = se.indexes.Set(key, pageID) + if err != nil { + return err + } err = se.savePage(pageID, *page) if err != nil { @@ -330,9 +352,9 @@ func (se *StorageEngine) Modify(key, value []byte) error { // Get 获取一个键对应的值 func (se *StorageEngine) Get(key []byte) ([]byte, error) { - index, ok := se.indexes[string(key)] - if !ok { - return nil, fmt.Errorf("key not found") + index, err := se.indexes.Get(key) + if err != nil { + return nil, err } page, exists := se.getPage(index) if !exists { @@ -353,9 +375,9 @@ func (se *StorageEngine) Delete(key []byte) error { se.Lock() defer se.Unlock() - index, ok := se.indexes[string(key)] - if !ok { - return fmt.Errorf("key not found") + index, err := se.indexes.Get(key) + if err != nil { + return err } page, exists := se.getPage(index) @@ -363,7 +385,7 @@ func (se *StorageEngine) Delete(key []byte) error { return fmt.Errorf("page not found") } - err := se.walOp([]byte(key), page.PageID, WalOpDelete) + err = se.walOp(key, page.PageID, WalOpDelete) if err != nil { return fmt.Errorf("failed to write wal: %w", err) } @@ -380,9 +402,12 @@ func (se *StorageEngine) Delete(key []byte) error { if found { page.UserRecords = newRecords - delete(se.indexes, string(key)) + err := se.indexes.Delete(key) + if err != nil { + return err + } - err := se.savePage(index, *page) + err = se.savePage(index, *page) if err != nil { return fmt.Errorf("failed to save to disk: %w", err) } @@ -390,58 +415,3 @@ func (se *StorageEngine) Delete(key []byte) error { return nil } - -func main() { - filePath := "data" - - engine, err := NewStorageEngine(filePath) - if err != nil { - fmt.Println("Error initializing storage engine:", err) - return - } - defer engine.Close() - - err = engine.Insert([]byte("name"), []byte("张三")) - if err != nil { - fmt.Println("Error inserting data:", err) - return - } - - err = engine.Insert([]byte("age"), []byte("25")) - if err != nil { - fmt.Println("Error inserting data:", err) - return - } - - if val, err := engine.Get([]byte("name")); err == nil { - fmt.Println("Name:", string(val)) - } else { - fmt.Println(err.Error()) - } - if val, err := engine.Get([]byte("age")); err == nil { - fmt.Println("Age:", string(val)) - } else { - fmt.Println(err.Error()) - } - err = engine.Modify([]byte("age"), []byte("35")) - if err != nil { - fmt.Println("Error Modify data:", err) - return - } - if val, err := engine.Get([]byte("age")); err == nil { - fmt.Println("Age:", string(val)) - } else { - fmt.Println(err.Error()) - } - - err = engine.Delete([]byte("age")) - if err != nil { - fmt.Println("Error deleting data:", err) - return - } - - if _, err := engine.Get([]byte("age")); err != nil { - fmt.Println("del age", err) - fmt.Println("Age deleted successfully") - } -} diff --git a/storage_test.go b/storage_test.go new file mode 100644 index 0000000..693ea31 --- /dev/null +++ b/storage_test.go @@ -0,0 +1,61 @@ +package main + +import ( + "testing" +) + +func TestEngine(t *testing.T) { + filePath := "data" + + opt := Options{DirPath: filePath} + engine, err := NewStorageEngine(opt) + if err != nil { + t.Log("Error initializing storage engine:", err) + return + } + defer engine.Close() + + err = engine.Insert([]byte("name"), []byte("张三")) + if err != nil { + t.Log("Error inserting data:", err) + return + } + + err = engine.Insert([]byte("age"), []byte("25")) + if err != nil { + t.Log("Error inserting data:", err) + return + } + + if val, err := engine.Get([]byte("name")); err == nil { + t.Log("Name:", string(val)) + } else { + t.Log(err.Error()) + } + if val, err := engine.Get([]byte("age")); err == nil { + t.Log("Age:", string(val)) + } else { + t.Log(err.Error()) + } + err = engine.Modify([]byte("age"), []byte("35")) + if err != nil { + t.Log("Error Modify data:", err) + return + } + if val, err := engine.Get([]byte("age")); err == nil { + t.Log("Age:", string(val)) + } else { + t.Log(err.Error()) + } + + err = engine.Delete([]byte("age")) + if err != nil { + t.Log("Error deleting data:", err) + return + } + + if _, err := engine.Get([]byte("age")); err != nil { + t.Log("del age", err) + t.Log("Age deleted successfully") + } +} -- Gitee From 7e0f5e3c8e1d36d58cd20d4124b61590bfb474d0 Mon Sep 17 00:00:00 2001 From: WSRer <1749094641@qq.com> Date: Tue, 15 Oct 2024 17:27:45 +0800 Subject: [PATCH 2/2] update --- index.go | 20 ++++++++++++++------ main.go | 9 +++++---- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/index.go b/index.go index 6370d5f..56fc09a 100644 --- a/index.go +++ b/index.go @@ -3,6 +3,7 @@ package main import ( "fmt" "iter" + "unsafe" ) type Index interface { @@ -18,11 +19,18 @@ type DefaultIndex struct { inner map[string]uint64 } +func stringToBytes(s string) []byte { + return unsafe.Slice(unsafe.StringData(s), len(s)) +} + +func bytesToString(b []byte) string { + return unsafe.String(&b[0], len(b)) +} + func (d *DefaultIndex) Iterator() iter.Seq2[[]byte, uint64] { return func(yield func([]byte, uint64) bool) { for k, v := range d.inner { - // 将字符串键转换为[]byte - keyBytes := []byte(k) + keyBytes := stringToBytes(k) if !yield(keyBytes, v) { return } @@ -31,7 +39,7 @@ func (d *DefaultIndex) Iterator() iter.Seq2[[]byte, uint64] { } func (d *DefaultIndex) Get(key []byte) (uint64, error) { - u, exists := d.inner[string(key)] + u, exists := d.inner[bytesToString(key)] if !exists { return 0, fmt.Errorf("key not found") } @@ -39,13 +47,13 @@ func (d *DefaultIndex) Get(key []byte) (uint64, error) { } func (d *DefaultIndex) Set(key []byte, value uint64) error { - d.inner[string(key)] = value + d.inner[bytesToString(key)] = value return nil } func (d *DefaultIndex) Delete(key []byte) error { - if _, exists := d.inner[string(key)]; exists { - delete(d.inner, string(key)) + if _, exists := d.inner[bytesToString(key)]; exists { + delete(d.inner, bytesToString(key)) } return nil } diff --git a/main.go b/main.go index 95f2665..f9f5689 100644 --- a/main.go +++ b/main.go @@ -9,6 +9,7 @@ import ( "io" "os" "path/filepath" + "slices" "sync" "github.com/rosedblabs/wal" @@ -292,7 +293,7 @@ func (se *StorageEngine) Insert(key, value []byte) error { } for _, record := range page.UserRecords { - if string(record.Key) == string(key) { + if slices.Equal(record.Key, key) { return fmt.Errorf("key already exists") } } @@ -332,7 +333,7 @@ func (se *StorageEngine) Modify(key, value []byte) error { } for idx, record := range page.UserRecords { - if string(record.Key) == string(key) { + if slices.Equal(record.Key, key) { page.UserRecords[idx].Value = value } } @@ -362,7 +363,7 @@ func (se *StorageEngine) Get(key []byte) ([]byte, error) { } for _, record := range page.UserRecords { - if string(record.Key) == string(key) { + if slices.Equal(record.Key, key) { return record.Value, nil } } @@ -393,7 +394,7 @@ func (se *StorageEngine) Delete(key []byte) error { newRecords := make([]Record, 0) found := false for _, record := range page.UserRecords { - if string(record.Key) == string(key) { + if slices.Equal(record.Key, key) { found = true continue } -- Gitee