Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

sql: changing name column type to binary type #1762

Merged
merged 3 commits into from
Apr 13, 2022
Merged

sql: changing name column type to binary type #1762

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bc1fdb2
change the name column type to binary type
zhijian-pro Apr 12, 2022
f4e81b2
adjust length of name
zhijian-pro Apr 12, 2022
dd552ed
refactor
zhijian-pro Apr 13, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions pkg/meta/dump.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,9 +132,9 @@ func parseHex(c byte) (byte, error) {
}
}

func unescape(s string) string {
func unescape(s string) []byte {
if !strings.ContainsRune(s, '%') {
return s
return []byte(s)
}

p := []byte(s)
Expand All @@ -152,7 +152,7 @@ func unescape(s string) string {
p[n] = c
n++
}
return string(p[:n])
return p[:n]
}

func (de *DumpedEntry) writeJSON(bw *bufio.Writer, depth int) error {
Expand Down
4 changes: 2 additions & 2 deletions pkg/meta/load_dump_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,13 @@ const sampleFile = "metadata.sample"
const subSampleFile = "metadata-sub.sample"

func TestEscape(t *testing.T) {
var cs []byte = []byte("hello 世界")
var cs = []byte("hello 世界")
for i := 0; i < 256; i++ {
cs = append(cs, byte(i))
}
s := string(cs)
r := unescape(escape(s))
if r != s {
if bytes.Compare(r, cs) != 0 {
t.Fatalf("expected %v, but got %v", s, r)
}
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/meta/redis.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3055,7 +3055,7 @@ func (m *redisMeta) loadEntry(e *DumpedEntry, cs *DumpedCounters, refs map[strin
if len(e.Entries) > 0 {
dentries := make(map[string]interface{})
for _, c := range e.Entries {
dentries[unescape(c.Name)] = m.packEntry(typeFromString(c.Attr.Type), c.Attr.Inode)
dentries[string(unescape(c.Name))] = m.packEntry(typeFromString(c.Attr.Type), c.Attr.Inode)
}
p.HSet(ctx, m.entryKey(inode), dentries)
}
Expand Down
82 changes: 29 additions & 53 deletions pkg/meta/sql.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ type counter struct {

type edge struct {
Parent Ino `xorm:"unique(edge) notnull"`
Name string `xorm:"unique(edge) notnull"`
Name []byte `xorm:"unique(edge) varbinary(255) notnull"`
Inode Ino `xorm:"notnull"`
Type uint8 `xorm:"notnull"`
}
Expand All @@ -77,7 +77,7 @@ type node struct {

type namedNode struct {
node `xorm:"extends"`
Name string
Name []byte `xorm:"varbinary(255)"`
}

type chunk struct {
Expand All @@ -92,7 +92,7 @@ type chunkRef struct {
}
type symlink struct {
Inode Ino `xorm:"pk"`
Target string `xorm:"varchar(4096) notnull"`
Target []byte `xorm:"varbinary(4096) notnull"`
}

type xattr struct {
Expand Down Expand Up @@ -208,25 +208,6 @@ func (m *dbMeta) doDeleteSlice(chunkid uint64, size uint32) error {
})
}

func (m *dbMeta) updateCollate() {
if r, err := m.db.Query("show create table jfs_edge"); err != nil {
logger.Fatalf("show table jfs_edge: %s", err.Error())
} else {
createTable := string(r[0]["Create Table"])
// the default collate is case-insensitive
if !strings.Contains(createTable, "SET utf8mb4 COLLATE utf8mb4_bin") {
_, err := m.db.Exec("alter table jfs_edge modify name varchar (255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL")
if err != nil && strings.Contains(err.Error(), "Error 1071: Specified key was too long; max key length is 767 bytes") {
// MySQL 5.6 supports key length up to 767 bytes, so reduce the length of name to 190 chars
_, err = m.db.Exec("alter table jfs_edge modify name varchar (190) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL")
}
if err != nil {
logger.Fatalf("update collate: %s", err)
}
}
}
}

func (m *dbMeta) Init(format Format, force bool) error {
if err := m.db.Sync2(new(setting), new(counter)); err != nil {
logger.Fatalf("create table setting, counter: %s", err)
Expand All @@ -246,9 +227,6 @@ func (m *dbMeta) Init(format Format, force bool) error {
if err := m.db.Sync2(new(flock), new(plock)); err != nil {
logger.Fatalf("create table flock, plock: %s", err)
}
if m.db.DriverName() == "mysql" {
m.updateCollate()
}

var s = setting{Name: "format"}
ok, err := m.db.Get(&s)
Expand Down Expand Up @@ -358,9 +336,6 @@ func (m *dbMeta) doNewSession(sinfo []byte) error {
if err = m.db.Sync2(new(flock), new(plock)); err != nil {
return fmt.Errorf("update table flock, plock: %s", err)
}
if m.db.DriverName() == "mysql" {
m.updateCollate()
}

for {
if err = m.txn(func(s *xorm.Session) error {
Expand Down Expand Up @@ -624,7 +599,7 @@ func (m *dbMeta) doLookup(ctx Context, parent Ino, name string, inode *Ino, attr
if attr != nil {
dbSession = dbSession.Join("INNER", &node{}, "jfs_edge.inode=jfs_node.inode")
}
nn := namedNode{node: node{Parent: parent}, Name: name}
nn := namedNode{node: node{Parent: parent}, Name: []byte(name)}
exist, err := dbSession.Select("*").Get(&nn)
if err != nil {
return errno(err)
Expand Down Expand Up @@ -982,7 +957,7 @@ func (m *dbMeta) doMknod(ctx Context, parent Ino, name string, _type uint8, mode
if pn.Type != TypeDirectory {
return syscall.ENOTDIR
}
var e = edge{Parent: parent, Name: name}
var e = edge{Parent: parent, Name: []byte(name)}
ok, err = s.Get(&e)
if err != nil {
return err
Expand Down Expand Up @@ -1030,7 +1005,7 @@ func (m *dbMeta) doMknod(ctx Context, parent Ino, name string, _type uint8, mode
}
}

if err = mustInsert(s, &edge{parent, name, ino, _type}, &n); err != nil {
if err = mustInsert(s, &edge{parent, []byte(name), ino, _type}, &n); err != nil {
return err
}
if parent != TrashInode {
Expand All @@ -1039,7 +1014,7 @@ func (m *dbMeta) doMknod(ctx Context, parent Ino, name string, _type uint8, mode
}
}
if _type == TypeSymlink {
if err = mustInsert(s, &symlink{Inode: ino, Target: path}); err != nil {
if err = mustInsert(s, &symlink{Inode: ino, Target: []byte(path)}); err != nil {
return err
}
}
Expand Down Expand Up @@ -1072,15 +1047,15 @@ func (m *dbMeta) doUnlink(ctx Context, parent Ino, name string) syscall.Errno {
if pn.Type != TypeDirectory {
return syscall.ENOTDIR
}
var e = edge{Parent: parent, Name: name}
var e = edge{Parent: parent, Name: []byte(name)}
ok, err = s.Get(&e)
if err != nil {
return err
}
if !ok && m.conf.CaseInsensi {
if ee := m.resolveCase(ctx, parent, name); ee != nil {
ok = true
e.Name = string(ee.Name)
e.Name = ee.Name
e.Inode = ee.Inode
e.Type = ee.Attr.Typ
}
Expand Down Expand Up @@ -1134,7 +1109,7 @@ func (m *dbMeta) doUnlink(ctx Context, parent Ino, name string) syscall.Errno {
return err
}
if trash > 0 {
if err = mustInsert(s, &edge{trash, fmt.Sprintf("%d-%d-%s", parent, e.Inode, e.Name), e.Inode, e.Type}); err != nil {
if err = mustInsert(s, &edge{trash, []byte(fmt.Sprintf("%d-%d-%s", parent, e.Inode, string(e.Name))), e.Inode, e.Type}); err != nil {
return err
}
}
Expand Down Expand Up @@ -1200,7 +1175,7 @@ func (m *dbMeta) doRmdir(ctx Context, parent Ino, name string) syscall.Errno {
if pn.Type != TypeDirectory {
return syscall.ENOTDIR
}
var e = edge{Parent: parent, Name: name}
var e = edge{Parent: parent, Name: []byte(name)}
ok, err = s.Get(&e)
if err != nil {
return err
Expand All @@ -1209,7 +1184,7 @@ func (m *dbMeta) doRmdir(ctx Context, parent Ino, name string) syscall.Errno {
if ee := m.resolveCase(ctx, parent, name); ee != nil {
ok = true
e.Inode = ee.Inode
e.Name = string(ee.Name)
e.Name = ee.Name
e.Type = ee.Attr.Typ
}
}
Expand Down Expand Up @@ -1256,7 +1231,7 @@ func (m *dbMeta) doRmdir(ctx Context, parent Ino, name string) syscall.Errno {
if _, err = s.Cols("ctime", "parent").Update(&n, &node{Inode: n.Inode}); err != nil {
return err
}
if err = mustInsert(s, &edge{trash, fmt.Sprintf("%d-%d-%s", parent, e.Inode, e.Name), e.Inode, e.Type}); err != nil {
if err = mustInsert(s, &edge{trash, []byte(fmt.Sprintf("%d-%d-%s", parent, e.Inode, string(e.Name))), e.Inode, e.Type}); err != nil {
return err
}
} else {
Expand Down Expand Up @@ -1289,7 +1264,7 @@ func (m *dbMeta) doRename(ctx Context, parentSrc Ino, nameSrc string, parentDst
var dn node
var newSpace, newInode int64
err := m.txn(func(s *xorm.Session) error {
var se = edge{Parent: parentSrc, Name: nameSrc}
var se = edge{Parent: parentSrc, Name: []byte(nameSrc)}
ok, err := s.Get(&se)
if err != nil {
return err
Expand All @@ -1299,13 +1274,13 @@ func (m *dbMeta) doRename(ctx Context, parentSrc Ino, nameSrc string, parentDst
ok = true
se.Inode = e.Inode
se.Type = e.Attr.Typ
se.Name = string(e.Name)
se.Name = e.Name
}
}
if !ok {
return syscall.ENOENT
}
if parentSrc == parentDst && se.Name == nameDst {
if parentSrc == parentDst && string(se.Name) == nameDst {
if inode != nil {
*inode = se.Inode
}
Expand Down Expand Up @@ -1342,7 +1317,7 @@ func (m *dbMeta) doRename(ctx Context, parentSrc Ino, nameSrc string, parentDst
return syscall.ENOENT
}

var de = edge{Parent: parentDst, Name: nameDst}
var de = edge{Parent: parentDst, Name: []byte(nameDst)}
ok, err = s.Get(&de)
if err != nil {
return err
Expand All @@ -1352,7 +1327,7 @@ func (m *dbMeta) doRename(ctx Context, parentSrc Ino, nameSrc string, parentDst
ok = true
de.Inode = e.Inode
de.Type = e.Attr.Typ
de.Name = string(e.Name)
de.Name = e.Name
}
}
now := time.Now().UnixNano() / 1e3
Expand Down Expand Up @@ -1453,7 +1428,7 @@ func (m *dbMeta) doRename(ctx Context, parentSrc Ino, nameSrc string, parentDst
return err
}
name := fmt.Sprintf("%d-%d-%s", parentDst, dino, de.Name)
if err = mustInsert(s, &edge{trash, name, dino, de.Type}); err != nil {
if err = mustInsert(s, &edge{trash, []byte(name), dino, de.Type}); err != nil {
return err
}
} else if de.Type != TypeDirectory && dn.Nlink > 0 {
Expand Down Expand Up @@ -1536,7 +1511,7 @@ func (m *dbMeta) doLink(ctx Context, inode, parent Ino, name string, attr *Attr)
if pn.Type != TypeDirectory {
return syscall.ENOTDIR
}
var e = edge{Parent: parent, Name: name}
var e = edge{Parent: parent, Name: []byte(name)}
ok, err = s.Get(&e)
if err != nil {
return err
Expand All @@ -1563,7 +1538,7 @@ func (m *dbMeta) doLink(ctx Context, inode, parent Ino, name string, attr *Attr)
n.Nlink++
n.Ctime = now

if err = mustInsert(s, &edge{Parent: parent, Name: name, Inode: inode, Type: n.Type}); err != nil {
if err = mustInsert(s, &edge{Parent: parent, Name: []byte(name), Inode: inode, Type: n.Type}); err != nil {
return err
}
if _, err := s.Cols("mtime", "ctime").Update(&pn, &node{Inode: parent}); err != nil {
Expand All @@ -1589,13 +1564,13 @@ func (m *dbMeta) doReaddir(ctx Context, inode Ino, plus uint8, entries *[]*Entry
return errno(err)
}
for _, n := range nodes {
if n.Name == "" {
if len(n.Name) == 0 {
logger.Errorf("Corrupt entry with empty name: inode %d parent %d", n.Inode, inode)
continue
}
entry := &Entry{
Inode: n.Inode,
Name: []byte(n.Name),
Name: n.Name,
Attr: &Attr{},
}
if plus != 0 {
Expand Down Expand Up @@ -2333,7 +2308,7 @@ func (m *dbMeta) dumpEntry(inode Ino, typ uint8) (*DumpedEntry, error) {
if !ok {
logger.Warnf("no link target for inode %d", inode)
}
e.Symlink = l.Target
e.Symlink = string(l.Target)
}
return nil
})
Expand Down Expand Up @@ -2382,7 +2357,7 @@ func (m *dbMeta) dumpEntryFast(inode Ino, typ uint8) *DumpedEntry {
logger.Warnf("no link target for inode %d", inode)
l = &symlink{}
}
e.Symlink = l.Target
e.Symlink = string(l.Target)
}
return e
}
Expand All @@ -2409,7 +2384,8 @@ func (m *dbMeta) dumpDir(inode Ino, tree *DumpedEntry, bw *bufio.Writer, depth i
if err := tree.writeJsonWithOutEntry(bw, depth); err != nil {
return err
}
sort.Slice(edges, func(i, j int) bool { return edges[i].Name < edges[j].Name })

sort.Slice(edges, func(i, j int) bool { return bytes.Compare(edges[i].Name, edges[j].Name) == -1 })

for idx, e := range edges {
var entry *DumpedEntry
Expand All @@ -2426,7 +2402,7 @@ func (m *dbMeta) dumpDir(inode Ino, tree *DumpedEntry, bw *bufio.Writer, depth i
continue
}

entry.Name = e.Name
entry.Name = string(e.Name)
if e.Type == TypeDirectory {
err = m.dumpDir(e.Inode, entry, bw, depth+2, showProgress)
} else {
Expand Down Expand Up @@ -2713,7 +2689,7 @@ func (m *dbMeta) loadEntry(e *DumpedEntry, cs *DumpedCounters, refs map[uint64]*
if len(e.Xattrs) > 0 {
xattrs := make([]*xattr, 0, len(e.Xattrs))
for _, x := range e.Xattrs {
xattrs = append(xattrs, &xattr{inode, x.Name, []byte(unescape(x.Value))})
xattrs = append(xattrs, &xattr{inode, x.Name, unescape(x.Value)})
}
beans = append(beans, xattrs)
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/meta/tkv.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2314,7 +2314,7 @@ func (m *kvMeta) loadEntry(e *DumpedEntry, cs *DumpedCounters, refs map[string]i
} else if attr.Typ == TypeDirectory {
attr.Length = 4 << 10
for _, c := range e.Entries {
tx.set(m.entryKey(inode, unescape(c.Name)), m.packEntry(typeFromString(c.Attr.Type), c.Attr.Inode))
tx.set(m.entryKey(inode, string(unescape(c.Name))), m.packEntry(typeFromString(c.Attr.Type), c.Attr.Inode))
}
} else if attr.Typ == TypeSymlink {
symL := unescape(e.Symlink)
Expand Down