1 Star 0 Fork 0

huangsg / rsc

Create your Gitee Account
Explore and code with more than 6 million developers,Free private repositories !:)
Sign up
Clone or download
app.go 22.53 KB
Copy Edit Raw Blame History
huangsg authored 2020-03-01 20:37 . 初始化代码
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package server implements an appfs server backed by the
// App Engine datastore.
package server
import (
"bytes"
"crypto/sha1"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/http"
"path"
"strconv"
"strings"
"time"
"appengine"
"appengine/datastore"
"appengine/memcache"
"appengine/user"
"gitee.com/hsg4ok_admin/rsc/appfs/fs"
"gitee.com/hsg4ok_admin/rsc/appfs/proto"
)
const pwFile = "/.password"
var chatty = false
func init() {
handle(proto.ReadURL, (*request).read)
handle(proto.WriteURL, (*request).write)
handle(proto.StatURL, (*request).stat)
handle(proto.MkfsURL, (*request).mkfs)
handle(proto.CreateURL, (*request).create)
handle(proto.RemoveURL, (*request).remove)
}
type request struct {
w http.ResponseWriter
req *http.Request
c appengine.Context
name string
mname string
key *datastore.Key
}
func auth(r *request) bool {
hdr := r.req.Header.Get("Authorization")
if !strings.HasPrefix(hdr, "Basic ") {
return false
}
data, err := base64.StdEncoding.DecodeString(hdr[6:])
if err != nil {
return false
}
i := bytes.IndexByte(data, ':')
if i < 0 {
return false
}
user, passwd := string(data[:i]), string(data[i+1:])
_, data, err = read(r.c, pwFile)
if err != nil {
r.c.Errorf("reading %s: %v", pwFile, err)
if _, err := mkfs(r.c); err != nil {
r.c.Errorf("creating fs: %v", err)
}
_, data, err = read(r.c, pwFile)
if err != nil {
r.c.Errorf("reading %s again: %v", pwFile, err)
return false
}
}
lines := strings.Split(string(data), "\n")
for _, line := range lines {
if strings.HasPrefix(line, "#") {
continue
}
f := strings.Fields(line)
if len(f) < 3 {
continue
}
if f[0] == user {
return hash(f[1]+passwd) == f[2]
}
}
return false
}
func hash(s string) string {
h := sha1.New()
h.Write([]byte(s))
return fmt.Sprintf("%x", h.Sum(nil))
}
func handle(prefix string, f func(*request)) {
http.HandleFunc(prefix, func(w http.ResponseWriter, req *http.Request) {
c := appengine.NewContext(req)
r := &request{
w: w,
req: req,
c: c,
}
if strings.HasSuffix(prefix, "/") {
r.name, r.mname, r.key = mangle(c, req.URL.Path[len(prefix)-1:])
} else {
req.URL.Path = "/"
}
defer func() {
if err := recover(); err != nil {
w.WriteHeader(http.StatusConflict)
fmt.Fprintf(w, "%s\n", err)
}
}()
if !auth(r) {
w.Header().Set("WWW-Authenticate", "Basic realm=\"appfs\"")
http.Error(w, "Need auth", http.StatusUnauthorized)
return
}
f(r)
})
}
func mangle(c appengine.Context, name string) (string, string, *datastore.Key) {
name = path.Clean("/" + name)
n := strings.Count(name, "/")
if name == "/" {
n = 0
}
mname := fmt.Sprintf("%d%s", n, name)
root := datastore.NewKey(c, "RootKey", "v2:", 0, nil)
key := datastore.NewKey(c, "FileInfo", mname, 0, root)
return name, mname, key
}
type FileInfo struct {
Path string // mangled path
Name string
Qid int64 // assigned unique id number
Seq int64 // modification sequence number in file tree
ModTime time.Time
Size int64
IsDir bool
}
type FileData struct {
Data []byte
}
func stat(c appengine.Context, name string) (*FileInfo, error) {
var fi FileInfo
name, _, key := mangle(c, name)
c.Infof("DATASTORE Stat %q", name)
err := datastore.Get(c, key, &fi)
if err != nil {
return nil, err
}
return &fi, nil
}
func (r *request) saveStat(fi *FileInfo) {
jfi, err := json.Marshal(&fi)
if err != nil {
panic(err)
}
r.w.Header().Set("X-Appfs-Stat", string(jfi))
}
func (r *request) tx(f func(c appengine.Context) error) {
err := datastore.RunInTransaction(r.c, f, &datastore.TransactionOptions{XG: true})
if err != nil {
panic(err)
}
}
func (r *request) stat() {
var fi *FileInfo
r.tx(func(c appengine.Context) error {
fi1, err := stat(c, r.name)
if err != nil {
return err
}
fi = fi1
return nil
})
jfi, err := json.Marshal(&fi)
if err != nil {
panic(err)
}
r.w.Write(jfi)
}
func read(c appengine.Context, name string) (fi *FileInfo, data []byte, err error) {
name, _, _ = mangle(c, name)
fi1, err := stat(c, name)
if err != nil {
return nil, nil, err
}
if fi1.IsDir {
dt, err := readdir(c, name)
if err != nil {
return nil, nil, err
}
fi = fi1
data = dt
return fi, data, nil
}
root := datastore.NewKey(c, "RootKey", "v2:", 0, nil)
dkey := datastore.NewKey(c, "FileData", "", fi1.Qid, root)
var fd FileData
c.Infof("DATASTORE Read %q", name)
if err := datastore.Get(c, dkey, &fd); err != nil {
return nil, nil, err
}
fi = fi1
data = fd.Data
return fi, data, nil
}
func (r *request) read() {
var (
fi *FileInfo
data []byte
)
r.tx(func(c appengine.Context) error {
var err error
fi, data, err = read(r.c, r.name)
return err
})
r.saveStat(fi)
r.w.Write(data)
}
func readdir(c appengine.Context, name string) ([]byte, error) {
name, _, _ = mangle(c, name)
var buf bytes.Buffer
n := strings.Count(name, "/")
if name == "/" {
name = ""
n = 0
}
root := datastore.NewKey(c, "RootKey", "v2:", 0, nil)
first := fmt.Sprintf("%d%s/", n+1, name)
limit := fmt.Sprintf("%d%s0", n+1, name)
c.Infof("DATASTORE ReadDir %q", name)
q := datastore.NewQuery("FileInfo").
Filter("Path >=", first).
Filter("Path <", limit).
Ancestor(root)
enc := json.NewEncoder(&buf)
it := q.Run(c)
var fi FileInfo
var pfi proto.FileInfo
for {
fi = FileInfo{}
_, err := it.Next(&fi)
if err != nil {
if err == datastore.Done {
break
}
return nil, err
}
pfi = proto.FileInfo{
Name: fi.Name,
ModTime: fi.ModTime,
Size: fi.Size,
IsDir: fi.IsDir,
}
if err := enc.Encode(&pfi); err != nil {
return nil, err
}
}
return buf.Bytes(), nil
}
func readdirRaw(c appengine.Context, name string) ([]proto.FileInfo, error) {
name, _, _ = mangle(c, name)
n := strings.Count(name, "/")
if name == "/" {
name = ""
n = 0
}
root := datastore.NewKey(c, "RootKey", "v2:", 0, nil)
first := fmt.Sprintf("%d%s/", n+1, name)
limit := fmt.Sprintf("%d%s0", n+1, name)
c.Infof("DATASTORE ReadDir %q", name)
q := datastore.NewQuery("FileInfo").
Filter("Path >=", first).
Filter("Path <", limit).
Ancestor(root)
it := q.Run(c)
var fi FileInfo
var pfi proto.FileInfo
var out []proto.FileInfo
for {
fi = FileInfo{}
_, err := it.Next(&fi)
if err != nil {
if err == datastore.Done {
break
}
return nil, err
}
pfi = proto.FileInfo{
Name: fi.Name,
ModTime: fi.ModTime,
Size: fi.Size,
IsDir: fi.IsDir,
}
out = append(out, pfi)
}
println("READDIR", name, len(out))
return out, nil
}
var initPasswd = `# Password file
# This file controls access to the server.
# The format is lines of space-separated fields:
# user salt pwhash
# The pwhash is the SHA1 of the salt string concatenated with the password.
# user=dummy password=dummy (replace with your own entries)
dummy 12345 faa863c7d3d41893f80165c704b714d5e31bdd3b
`
func (r *request) mkfs() {
var fi *FileInfo
r.tx(func(c appengine.Context) error {
var err error
fi, err = mkfs(c)
return err
})
r.saveStat(fi)
}
func mkfs(c appengine.Context) (fi *FileInfo, err error) {
fi1, err := stat(c, "/")
if err == nil {
return fi1, nil
}
// Root needs to be created.
// Probably root key does too.
root := datastore.NewKey(c, "RootKey", "v2:", 0, nil)
_, err = datastore.Put(c, root, &struct{}{})
if err != nil {
return nil, fmt.Errorf("mkfs put root: %s", err)
}
// Entry for /.
_, mpath, key := mangle(c, "/")
fi3 := FileInfo{
Path: mpath,
Name: "/",
Seq: 2, // 2, not 1, because we're going to write password file with #2
Qid: 1,
ModTime: time.Now(),
Size: 0,
IsDir: true,
}
_, err = datastore.Put(c, key, &fi3)
if err != nil {
return nil, fmt.Errorf("mkfs put /: %s", err)
}
/*
* Would like to use this code but App Engine apparently
* does not let Get observe the effect of a Put in the same
* transaction. What planet does that make sense on?
* Instead, we have to execute just the datastore writes that this
* sequence would.
*
_, err = create(c, pwFile, false)
if err != nil {
return nil, fmt.Errorf("mkfs create .password: %s", err)
}
_, err = write(c, pwFile, []byte(initPasswd))
if err != nil {
return nil, fmt.Errorf("mkfs write .password: %s", err)
}
*
*/
{
name, mname, key := mangle(c, pwFile)
// Create data object.
dataKey := int64(2)
root := datastore.NewKey(c, "RootKey", "v2:", 0, nil)
dkey := datastore.NewKey(c, "FileData", "", dataKey, root)
_, err := datastore.Put(c, dkey, &FileData{[]byte(initPasswd)})
if err != nil {
return nil, err
}
// Create new directory entry.
_, elem := path.Split(name)
fi1 = &FileInfo{
Path: mname,
Name: elem,
Qid: 2,
Seq: 2,
ModTime: time.Now(),
Size: int64(len(initPasswd)),
IsDir: false,
}
if _, err := datastore.Put(c, key, fi1); err != nil {
return nil, err
}
}
return &fi3, nil
}
func (r *request) write() {
data, err := ioutil.ReadAll(r.req.Body)
if err != nil {
panic(err)
}
var fi *FileInfo
var seq int64
r.tx(func(c appengine.Context) error {
var err error
fi, seq, err = write(r.c, r.name, data)
return err
})
updateCacheTime(r.c, seq)
r.saveStat(fi)
}
func write(c appengine.Context, name string, data []byte) (*FileInfo, int64, error) {
name, _, key := mangle(c, name)
// Check that file exists and is not a directory.
fi1, err := stat(c, name)
if err != nil {
return nil, 0, err
}
if fi1.IsDir {
return nil, 0, fmt.Errorf("cannot write to directory")
}
// Fetch and increment root sequence number.
rfi, err := stat(c, "/")
if err != nil {
return nil, 0, err
}
rfi.Seq++
// Write data.
root := datastore.NewKey(c, "RootKey", "v2:", 0, nil)
dkey := datastore.NewKey(c, "FileData", "", fi1.Qid, root)
fd := &FileData{data}
if _, err := datastore.Put(c, dkey, fd); err != nil {
return nil, 0, err
}
// Update directory entry.
fi1.Seq = rfi.Seq
fi1.Size = int64(len(data))
fi1.ModTime = time.Now()
if _, err := datastore.Put(c, key, fi1); err != nil {
return nil, 0, err
}
// Update sequence numbers all the way to the root.
if err := updateSeq(c, name, rfi.Seq, 1); err != nil {
return nil, 0, err
}
return fi1, rfi.Seq, nil
}
func updateSeq(c appengine.Context, name string, seq int64, skip int) error {
p := path.Clean(name)
for i := 0; ; i++ {
if i >= skip {
_, _, key := mangle(c, p)
var fi FileInfo
if err := datastore.Get(c, key, &fi); err != nil {
return err
}
fi.Seq = seq
if _, err := datastore.Put(c, key, &fi); err != nil {
return err
}
}
if p == "/" {
break
}
p, _ = path.Split(p)
p = path.Clean(p)
}
return nil
}
func (r *request) remove() {
panic("remove not implemented")
}
func (r *request) create() {
var fi *FileInfo
var seq int64
isDir := r.req.FormValue("dir") == "1"
r.tx(func(c appengine.Context) error {
var err error
fi, seq, err = create(r.c, r.name, isDir, nil)
return err
})
updateCacheTime(r.c, seq)
r.saveStat(fi)
}
func create(c appengine.Context, name string, isDir bool, data []byte) (*FileInfo, int64, error) {
name, mname, key := mangle(c, name)
// File must not exist.
fi1, err := stat(c, name)
if err == nil {
return nil, 0, fmt.Errorf("file already exists")
}
if err != datastore.ErrNoSuchEntity {
return nil, 0, err
}
// Parent must exist and be a directory.
p, _ := path.Split(name)
fi2, err := stat(c, p)
if err != nil {
if err == datastore.ErrNoSuchEntity {
return nil, 0, fmt.Errorf("parent directory %q does not exist", p)
}
return nil, 0, err
}
if !fi2.IsDir {
return nil, 0, fmt.Errorf("parent %q is not a directory", p)
}
// Fetch and increment root sequence number.
rfi, err := stat(c, "/")
if err != nil {
return nil, 0, err
}
rfi.Seq++
var dataKey int64
// Create data object.
if !isDir {
dataKey = rfi.Seq
root := datastore.NewKey(c, "RootKey", "v2:", 0, nil)
dkey := datastore.NewKey(c, "FileData", "", dataKey, root)
_, err := datastore.Put(c, dkey, &FileData{data})
if err != nil {
return nil, 0, err
}
}
// Create new directory entry.
_, elem := path.Split(name)
fi1 = &FileInfo{
Path: mname,
Name: elem,
Qid: rfi.Seq,
Seq: rfi.Seq,
ModTime: time.Now(),
Size: int64(len(data)),
IsDir: isDir,
}
if _, err := datastore.Put(c, key, fi1); err != nil {
return nil, 0, err
}
// Update sequence numbers all the way to root,
// but skip entry we just wrote.
if err := updateSeq(c, name, rfi.Seq, 1); err != nil {
return nil, 0, err
}
return fi1, rfi.Seq, nil
}
// Implementation of fs.AppEngine.
func init() {
fs.Register(ae{})
}
type ae struct{}
func tx(c interface{}, f func(c appengine.Context) error) error {
return datastore.RunInTransaction(c.(appengine.Context), f, &datastore.TransactionOptions{XG: true})
}
func (ae) NewContext(req *http.Request) interface{} {
return appengine.NewContext(req)
}
func (ae) User(ctxt interface{}) string {
c := ctxt.(appengine.Context)
u := user.Current(c)
if u == nil {
return "?"
}
return u.String()
}
type cacheKey struct {
t int64
name string
}
func (ae) CacheRead(ctxt interface{}, name, path string) (key interface{}, data []byte, found bool) {
c := ctxt.(appengine.Context)
t, data, _, err := cacheRead(c, "cache", name, path)
return &cacheKey{t, name}, data, err == nil
}
func (ae) CacheWrite(ctxt, key interface{}, data []byte) {
c := ctxt.(appengine.Context)
k := key.(*cacheKey)
cacheWrite(c, k.t, "cache", k.name, data)
}
func (ae ae) Read(ctxt interface{}, name string) (data []byte, pfi *proto.FileInfo, err error) {
c := ctxt.(appengine.Context)
name = path.Clean("/"+name)
if chatty {
c.Infof("AE Read %s", name)
}
_, data, pfi, err = cacheRead(c, "data", name, name)
if err != nil {
err = fmt.Errorf("Read %q: %v", name, err)
}
return
}
func (ae) Write(ctxt interface{}, path string, data []byte) error {
var seq int64
err := tx(ctxt, func(c appengine.Context) error {
_, err := stat(c, path)
if err != nil {
_, seq, err = create(c, path, false, data)
} else {
_, seq, err = write(c, path, data)
}
return err
})
if seq != 0 {
updateCacheTime(ctxt.(appengine.Context), seq)
}
if err != nil {
err = fmt.Errorf("Write %q: %v", path, err)
}
return err
}
func (ae) Remove(ctxt interface{}, path string) error {
return fmt.Errorf("remove not implemented")
}
func (ae) Mkdir(ctxt interface{}, path string) error {
var seq int64
err := tx(ctxt, func(c appengine.Context) error {
var err error
_, seq, err = create(c, path, true, nil)
return err
})
if seq != 0 {
updateCacheTime(ctxt.(appengine.Context), seq)
}
if err != nil {
err = fmt.Errorf("Mkdir %q: %v", path, err)
}
return err
}
func (ae) Criticalf(ctxt interface{}, format string, args ...interface{}) {
ctxt.(appengine.Context).Criticalf(format, args...)
}
type readDirCacheEntry struct {
Dir []proto.FileInfo
Error string
}
func (ae) ReadDir(ctxt interface{}, name string) (dir []proto.FileInfo, err error) {
c := ctxt.(appengine.Context)
name = path.Clean("/"+name)
t, data, _, err := cacheRead(c, "dir", name, name)
if err == nil {
var e readDirCacheEntry
if err := json.Unmarshal(data, &e); err == nil {
if chatty {
c.Infof("cached ReadDir %q", name)
}
if e.Error != "" {
return nil, errors.New(e.Error)
}
return e.Dir, nil
}
c.Criticalf("unmarshal cached dir %q: %v", name)
}
err = tx(ctxt, func(c appengine.Context) error {
var err error
dir, err = readdirRaw(c, name)
return err
})
var e readDirCacheEntry
e.Dir = dir
if err != nil {
err = fmt.Errorf("ReadDir %q: %v", name, err)
e.Error = err.Error()
}
if data, err := json.Marshal(&e); err != nil {
c.Criticalf("json marshal cached dir: %v", err)
} else {
c.Criticalf("caching dir %q@%d %d bytes", name, t, len(data))
cacheWrite(c, t, "dir", name, data)
}
return
}
// Caching of file system data.
//
// The cache stores entries under keys of the form time,space,name,
// where time is the time at which the entry is valid for, space is a name
// space identifier, and name is an arbitrary name.
//
// A key of the form t,mtime,path maps to an integer value giving the
// modification time of the named path at root time t.
// The special key 0,mtime,/ is an integer giving the current time at the root.
//
// A key of the form t,data,path maps to the content of path at time t.
//
// Thus, a read from path should first obtain the root time,
// then obtain the modification time for the path at that root time
// then obtain the data for that path.
// t1 = get(0,mtime,/)
// t2 = get(t1,mtime,path)
// data = get(t2,data,path)
//
// The API allows clients to cache their own data too, with expiry tied to
// the modification time of a particular path (file or directory). To look
// up one of those, we use:
// t1 = get(0,mtime,/)
// t2 = get(t1,mtime,path)
// data = get(t2,clientdata,name)
//
// To store data in the cache, the t1, t2 should be determined before reading
// from datastore. Then the data should be saved under t2. This ensures
// that if a datastore update happens after the read but before the cache write,
// we'll be writing to an entry that will no longer be used (t2).
const rootMemcacheKey = "0,mtime,/"
func updateCacheTime(c appengine.Context, seq int64) {
const key = rootMemcacheKey
bseq := []byte(strconv.FormatInt(seq, 10))
for tries := 0; tries < 10; tries++ {
item, err := memcache.Get(c, key)
if err != nil {
c.Infof("memcache.Get %q: %v", key, err)
err = memcache.Add(c, &memcache.Item{Key: key, Value: bseq})
if err == nil {
c.Infof("memcache.Add %q %q ok", key, bseq)
return
}
c.Infof("memcache.Add %q %q: %v", key, bseq, err)
}
v, err := strconv.ParseInt(string(item.Value), 10, 64)
if err != nil {
c.Criticalf("memcache.Get %q = %q (%v)", key, item.Value, err)
return
}
if v >= seq {
return
}
item.Value = bseq
err = memcache.CompareAndSwap(c, item)
if err == nil {
c.Infof("memcache.CAS %q %d->%d ok", key, v, seq)
return
}
c.Infof("memcache.CAS %q %d->%d: %v", key, v, seq, err)
}
c.Criticalf("repeatedly failed to update root key")
}
func cacheTime(c appengine.Context) (t int64, err error) {
const key = rootMemcacheKey
item, err := memcache.Get(c, key)
if err == nil {
v, err := strconv.ParseInt(string(item.Value), 10, 64)
if err == nil {
if chatty {
c.Infof("cacheTime %q = %v", key, v)
}
return v, nil
}
c.Criticalf("memcache.Get %q = %q (%v) - deleting", key, item.Value, err)
memcache.Delete(c, key)
}
fi, err := stat(c, "/")
if err != nil {
c.Criticalf("stat /: %v", err)
return 0, err
}
updateCacheTime(c, fi.Seq)
return fi.Seq, nil
}
func cachePathTime(c appengine.Context, path string) (t int64, err error) {
t, err = cacheTime(c)
if err != nil {
return 0, err
}
key := fmt.Sprintf("%d,mtime,%s", t, path)
item, err := memcache.Get(c, key)
if err == nil {
v, err := strconv.ParseInt(string(item.Value), 10, 64)
if err == nil {
if chatty {
c.Infof("cachePathTime %q = %v", key, v)
}
return v, nil
}
c.Criticalf("memcache.Get %q = %q (%v) - deleting", key, item.Value, err)
memcache.Delete(c, key)
}
var seq int64
if fi, err := stat(c, path); err == nil {
seq = fi.Seq
}
c.Infof("cachePathTime save %q = %v", key, seq)
item = &memcache.Item{Key: key, Value: []byte(strconv.FormatInt(seq, 10))}
if err := memcache.Set(c, item); err != nil {
c.Criticalf("memcache.Set %q %q: %v", key, item.Value, err)
}
return seq, nil
}
type statCacheEntry struct {
FileInfo *proto.FileInfo
Error string
}
func cacheRead(c appengine.Context, kind, name, path string) (mtime int64, data []byte, pfi *proto.FileInfo, err error) {
for tries := 0; tries < 10; tries++ {
t, err := cachePathTime(c, path)
if err != nil {
return 0, nil, nil, err
}
key := fmt.Sprintf("%d,%s,%s", t, kind, name)
item, err := memcache.Get(c, key)
var data []byte
if item != nil {
data = item.Value
}
if err != nil {
c.Infof("memcache miss %q %v", key, err)
} else if chatty {
c.Infof("memcache hit %q (%d bytes)", key, len(data))
}
if kind != "data" {
// Not a file; whatever memcache says is all we have.
return t, data, nil, err
}
// Load stat from cache (includes negative entry).
statkey := fmt.Sprintf("%d,stat,%s", t, name)
var st statCacheEntry
_, err = memcache.JSON.Get(c, statkey, &st)
if err == nil {
if st.Error != "" {
if chatty {
c.Infof("memcache hit stat error %q %q", statkey, st.Error)
}
err = errors.New(st.Error)
} else {
if chatty {
c.Infof("memcache hit stat %q", statkey)
}
}
if err != nil || data != nil {
return t, data, st.FileInfo, err
}
}
// Need stat, or maybe stat+data.
var fi *FileInfo
if data != nil {
c.Infof("stat %q", name)
fi, err = stat(c, name)
if err == nil && fi.Seq != t {
c.Criticalf("loaded %s but found stat %d", key, fi.Seq)
continue
}
} else {
c.Infof("read %q", name)
fi, data, err = read(c, name)
if err == nil && fi.Seq != t {
c.Infof("loaded %s but found read %d", key, fi.Seq)
t = fi.Seq
key = fmt.Sprintf("%d,data,%s", t, name)
statkey = fmt.Sprintf("%d,stat,%s", t, name)
}
// Save data to memcache.
if err == nil {
if true || chatty {
c.Infof("save data in memcache %q", key)
}
item := &memcache.Item{Key: key, Value: data}
if err := memcache.Set(c, item); err != nil {
c.Criticalf("failed to cache %s: %v", key, err)
}
}
}
// Cache stat, including error.
st = statCacheEntry{}
if fi != nil {
st.FileInfo = &proto.FileInfo{
Name: fi.Name,
ModTime: fi.ModTime,
Size: fi.Size,
IsDir: fi.IsDir,
}
}
if err != nil {
st.Error = err.Error()
// If this is a deadline exceeded, do not cache.
if strings.Contains(st.Error, "Canceled") || strings.Contains(st.Error, "Deadline") {
return t, data, st.FileInfo, err
}
}
if chatty {
c.Infof("save stat in memcache %q", statkey)
}
if err := memcache.JSON.Set(c, &memcache.Item{Key: statkey, Object: &st}); err != nil {
c.Criticalf("failed to cache %s: %v", statkey, err)
}
// Done!
return t, data, st.FileInfo, err
}
c.Criticalf("failed repeatedly in cacheRead")
return 0, nil, nil, errors.New("cacheRead loop failed")
}
func cacheWrite(c appengine.Context, t int64, kind, name string, data []byte) error {
mkey := fmt.Sprintf("%d,%s,%s", t, kind, name)
if true || chatty {
c.Infof("cacheWrite %s %d bytes", mkey, len(data))
}
err := memcache.Set(c, &memcache.Item{Key: mkey, Value: data})
if err != nil {
c.Criticalf("cacheWrite memcache.Set %q: %v", mkey, err)
}
return err
}

Comment ( 0 )

Sign in for post a comment

Go
1
https://gitee.com/hsg4ok_admin/rsc.git
git@gitee.com:hsg4ok_admin/rsc.git
hsg4ok_admin
rsc
rsc
v0.0.1

Search