summaryrefslogtreecommitdiff
path: root/store/page.go
diff options
context:
space:
mode:
Diffstat (limited to 'store/page.go')
-rw-r--r--store/page.go233
1 files changed, 105 insertions, 128 deletions
diff --git a/store/page.go b/store/page.go
index f8f8263..f654174 100644
--- a/store/page.go
+++ b/store/page.go
@@ -1,83 +1,125 @@
package store
+import "C"
import (
- "encoding/json"
+ "encoding/binary"
+ "fmt"
log "github.com/sirupsen/logrus"
+ "github.com/syndtr/goleveldb/leveldb"
"os"
- "strconv"
)
const pageSize = 64
-const balanceThreshold = 16
-
-// pageCreateIfNotExist creates a page variant's base directory if it does not exist.
-func (s *Store) pageCreateIfNotExist(variant string) {
- if !s.dir(s.PagesDir(variant)) {
- if err := os.MkdirAll(s.PagesDir(variant), s.PermissionDir); err != nil {
- log.Fatalf("Error while crating page variant %s base directory, %s", variant, err)
- os.Exit(1)
+
+// pageDB returns leveldb of page variant and creates it as required.
+func (s *Store) pageDB(variant string) *leveldb.DB {
+ if s.pageldb[variant] != nil {
+ return s.pageldb[variant]
+ } else {
+ if db, err := leveldb.OpenFile(s.PageVariantPath(variant), nil); err != nil {
+ s.fatalClose(fmt.Sprintf("Error opening leveldb for page variant %s, %s", variant, err))
+ } else {
+ s.pageldb[variant] = db
+ if _, err = db.Get([]byte("\000"), nil); err != nil {
+ log.Infof("Page variant %s created.", variant)
+ s.pageSetTotalCount(variant, 0)
+ }
+ return db
}
}
+ return nil
}
-// pageWrite writes payload of a page.
-func (s *Store) pageWrite(variant string, entry int, content []string) {
- if payload, err := json.Marshal(content); err != nil {
- log.Fatalf("Error while generating page %v of variant %s, %s", entry, variant, err)
- os.Exit(1)
+// pageDBDestroy destroys leveldb of page variant.
+func (s *Store) pageDBDestroy(variant string) {
+ if err := s.pageDB(variant).Close(); err != nil {
+ s.fatalClose(fmt.Sprintf("Error closing leveldb for page variant %s, %s", variant, err))
} else {
- s.pageCreateIfNotExist(variant)
- if err = os.WriteFile(s.PagePath(variant, entry), payload, s.PermissionFile); err != nil {
- log.Fatalf("Error while saving page %v, %s", entry, err)
- os.Exit(1)
- }
+ delete(s.pageldb, variant)
+ }
+
+ if err := os.RemoveAll(s.PageVariantPath(variant)); err != nil {
+ s.fatalClose(fmt.Sprintf("Error destroying page variant %s, %s", variant, err))
}
-}
-// pageLockString returns lock identifier of a page.
-func (s *Store) pageLockString(variant string, entry int) string {
- return "page_" + variant + "_" + strconv.Itoa(entry)
+ log.Infof("Page variant %s destroyed.", variant)
}
-// PageTotal returns total amount of pages.
-func (s *Store) PageTotal(variant string) int {
- s.pageCreateIfNotExist(variant)
- if entries, err := os.ReadDir(s.PagesDir(variant)); err != nil {
- log.Fatalf("Error while reading pages of variant %s, %s", variant, err)
- os.Exit(1)
+// pageGetTotalCount gets total count of a page variant.
+func (s *Store) pageGetTotalCount(variant string) uint64 {
+ db := s.pageDB(variant)
+
+ if payload, err := db.Get([]byte("\000"), nil); err != nil {
+ s.fatalClose(fmt.Sprintf("Error getting page variant %s total count, %s", variant, err))
} else {
- return len(entries)
+ return binary.LittleEndian.Uint64(payload)
}
+
return 0
}
-// Page returns all entries in a page.
-func (s *Store) Page(variant string, entry int) []string {
- return s.page(variant, entry, true)
+// pageSetTotalCount sets total count of a page variant.
+func (s *Store) pageSetTotalCount(variant string, value uint64) {
+ db := s.pageDB(variant)
+
+ payload := make([]byte, 8)
+ binary.LittleEndian.PutUint64(payload, value)
+
+ if err := db.Put([]byte("\000"), payload, nil); err != nil {
+ s.fatalClose(fmt.Sprintf("Error setting page variant %s total count, %s", variant, err))
+ }
+}
+
+// pageAdvanceTotalCount advances total count of a page variant.
+func (s *Store) pageAdvanceTotalCount(variant string) {
+ s.pageSetTotalCount(variant, s.pageGetTotalCount(variant)+1)
+}
+
+// pageReduceTotalCount reduces total count of a page variant.
+func (s *Store) pageReduceTotalCount(variant string) {
+ s.pageSetTotalCount(variant, s.pageGetTotalCount(variant)-1)
}
-// page returns all entries in a page with optional locking.
-func (s *Store) page(variant string, entry int, lock bool) []string {
- if !s.file(s.PagePath(variant, entry)) {
+// PageTotal returns total amount of pages.
+func (s *Store) PageTotal(variant string) int {
+ return (int(s.pageGetTotalCount(variant)) / pageSize) + 1
+}
+
+// Page returns all entries in a page.
+func (s *Store) Page(variant string, entry int) []string {
+ if entry >= s.PageTotal(variant) {
return nil
}
- if lock {
- s.getLock(s.pageLockString(variant, entry)).RLock()
- defer s.getLock(s.pageLockString(variant, entry)).RUnlock()
- }
+ var page []string
+ start := entry*pageSize + 1
+ end := start + pageSize
+ begin := false
- var flakes []string
- if payload, err := os.ReadFile(s.PagePath(variant, entry)); err != nil {
- log.Fatalf("Error while reading page %v of variant %s, %s", entry, variant, err)
- os.Exit(1)
- } else {
- if err = json.Unmarshal(payload, &flakes); err != nil {
- log.Fatalf("Error while parsing page %v of variant %s, %s", entry, variant, err)
- os.Exit(1)
+ db := s.pageDB(variant)
+
+ iter := db.NewIterator(nil, nil)
+ i := 0
+ for iter.Next() {
+ if i == end {
+ break
+ }
+ if begin {
+ page = append(page, string(iter.Key()))
+ } else {
+ if i >= start {
+ begin = true
+ }
}
+ i++
}
- return flakes
+ iter.Release()
+ if err := iter.Error(); err != nil {
+ log.Warnf("Error iterating page variant %s entry %v, %s", variant, entry, err)
+ return nil
+ }
+
+ return page
}
// PageImages returns all images in a page.
@@ -103,89 +145,24 @@ func (s *Store) PageInsert(variant, flake string) {
return
}
- entry := s.PageTotal(variant) - 1
- s.getLock(s.pageLockString(variant, entry)).Lock()
- defer s.getLock(s.pageLockString(variant, entry)).Unlock()
- var page []string
- if entry == -1 || len(s.page(variant, entry, false)) == pageSize {
- entry++
- page = []string{}
- } else {
- page = s.page(variant, entry, false)
+ s.getLock("page_" + variant).Lock()
+ defer s.getLock("page_" + variant).Unlock()
+
+ db := s.pageDB(variant)
+ if err := db.Put([]byte(flake), []byte{}, nil); err != nil {
+ s.fatalClose(fmt.Sprintf("Error inserting image %s into page variant %s, %s", flake, variant, err))
}
- page = append(page, flake)
- s.pageWrite(variant, entry, page)
+ s.pageAdvanceTotalCount(variant)
}
// PageRegisterRemove registers an image remove.
func (s *Store) PageRegisterRemove(variant, flake string) {
- go func() {
- var target int
- var targetPayload []string
- for i := 0; i < s.PageTotal(variant); i++ {
- page := s.page(variant, i, true)
- ok := false
- for j, f := range page {
- if f == flake {
- ok = true
- page = append(page[:(j)], page[(j+1):]...)
- break
- }
- }
- if ok {
- s.getLock(s.pageLockString(variant, i)).Lock()
- s.pageWrite(variant, i, page)
- s.getLock(s.pageLockString(variant, i)).Unlock()
- target = i
- targetPayload = page
- log.Infof("Removed page %v variant %s item %s.", target, variant, flake)
- break
- }
- }
- if len(targetPayload) < pageSize-balanceThreshold && (target+1) < s.PageTotal(variant) {
- log.Infof("Page %v reached balancing threshold, initiating balance.", target)
- s.PageBalance(variant, target, targetPayload)
- }
- }()
-}
+ s.getLock("page_" + variant).Lock()
+ defer s.getLock("page_" + variant).Unlock()
-// PageBalance balances a page.
-func (s *Store) PageBalance(variant string, target int, payload []string) {
- if !s.file(s.PagePath(variant, target)) {
- return
+ db := s.pageDB(variant)
+ if err := db.Delete([]byte(flake), nil); err != nil {
+ s.fatalClose(fmt.Sprintf("Error removing image %s from page variant %s, %s", flake, variant, err))
}
- go func() {
- if (target + 1) >= s.PageTotal(variant) {
- return
- }
- s.getLock(s.pageLockString(variant, target)).Lock()
- defer s.getLock(s.pageLockString(variant, target)).Unlock()
- s.getLock(s.pageLockString(variant, target+1)).Lock()
- defer s.getLock(s.pageLockString(variant, target+1)).Unlock()
- if payload == nil {
- payload = s.page(variant, target, false)
- }
- comp := pageSize - len(payload)
- next := s.page(variant, target+1, false)
- empty := false
- if len(next) <= comp {
- if err := os.Remove(s.PagePath(variant, target+1)); err != nil {
- log.Fatalf("Error removing empty page %v, %s", target+1, err)
- os.Exit(1)
- }
- payload = append(payload, next...)
- empty = true
- } else {
- payload = append(payload, next[:comp]...)
- next = next[comp+1:]
- }
-
- s.pageWrite(variant, target, payload)
- if !empty {
- s.pageWrite(variant, target+1, next)
- s.PageBalance(variant, target+1, next)
- } else {
- log.Infof("Removed empty page %v.", target+1)
- }
- }()
+ s.pageReduceTotalCount(variant)
}